From f19bb8c28054ec91be8fda2663d5d6594b292a49 Mon Sep 17 00:00:00 2001 From: Tyler Ganter Date: Wed, 8 Apr 2020 10:24:08 -0400 Subject: [PATCH 1/3] retrying --- voxel51/users/api.py | 58 ++++++++++++++++++-------------------------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/voxel51/users/api.py b/voxel51/users/api.py index 72d9645..4cfa45f 100644 --- a/voxel51/users/api.py +++ b/voxel51/users/api.py @@ -25,6 +25,7 @@ import dateutil.parser import mimetypes import requests +from retrying import retry import voxel51.users.auth as voxa import voxel51.users.jobs as voxj @@ -161,8 +162,7 @@ def list_analytics(self, all_versions=False): ''' endpoint = voxu.urljoin(self.base_url, "analytics", "list") data = {"all_versions": all_versions} - res = self._requests.get(endpoint, headers=self._header, json=data) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header, json=data) return _parse_json_response(res)["analytics"] def query_analytics(self, analytics_query): @@ -181,9 +181,8 @@ def query_analytics(self, analytics_query): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "analytics") - res = self._requests.get( + res = self._request_get( endpoint, headers=self._header, json=analytics_query.to_dict()) - _validate_response(res) return _parse_json_response(res) def get_analytic_id(self, name, version=None): @@ -236,8 +235,7 @@ def get_analytic_details(self, analytic_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "analytics", analytic_id) - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["analytic"] def get_analytic_doc(self, analytic_id): @@ -253,8 +251,7 @@ def get_analytic_doc(self, analytic_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "analytics", analytic_id, "doc") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res) def upload_analytic(self, doc_json_path, analytic_type=None): @@ -331,6 +328,7 @@ def delete_analytic(self, analytic_id): res = self._requests.delete(endpoint, headers=self._header) _validate_response(res) + def batch_get_analytic_details(self, analytic_ids): '''Gets details about the analytics with the given IDs. @@ -361,8 +359,7 @@ def list_data(self): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "data", "list") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["data"] def query_data(self, data_query): @@ -380,9 +377,8 @@ def query_data(self, data_query): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "data") - res = self._requests.get( + res = self._request_get( endpoint, headers=self._header, json=data_query.to_dict()) - _validate_response(res) return _parse_json_response(res) def upload_data(self, path, ttl=None): @@ -476,8 +472,7 @@ def get_data_details(self, data_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "data", data_id) - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["data"] def download_data(self, data_id, output_path=None): @@ -515,8 +510,7 @@ def get_data_download_url(self, data_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "data", data_id, "download-url") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["url"] def update_data_ttl(self, data_id, days=None, expiration_date=None): @@ -649,8 +643,7 @@ def list_jobs(self): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs", "list") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["jobs"] def query_jobs(self, jobs_query): @@ -668,9 +661,8 @@ def query_jobs(self, jobs_query): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs") - res = self._requests.get( + res = self._request_get( endpoint, headers=self._header, json=jobs_query.to_dict()) - _validate_response(res) return _parse_json_response(res) def upload_job_request( @@ -720,8 +712,7 @@ def get_job_details(self, job_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs", job_id) - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["job"] def get_job_request(self, job_id): @@ -738,8 +729,7 @@ def get_job_request(self, job_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs", job_id, "request") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return voxj.JobRequest.from_dict(_parse_json_response(res)) def start_job(self, job_id): @@ -927,8 +917,7 @@ def get_job_status(self, job_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs", job_id, "status") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res) def download_job_output(self, job_id, output_path=None): @@ -967,8 +956,7 @@ def get_job_output_download_url(self, job_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs", job_id, "output-url") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["url"] def download_job_logfile(self, job_id, output_path=None): @@ -990,8 +978,7 @@ def download_job_logfile(self, job_id, output_path=None): ''' endpoint = voxu.urljoin(self.base_url, "jobs", job_id, "log") if output_path is None: - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return res.content.decode() self._stream_download(endpoint, output_path) @@ -1014,8 +1001,7 @@ def get_job_logfile_download_url(self, job_id): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "jobs", job_id, "log-url") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["url"] def delete_job(self, job_id): @@ -1198,8 +1184,7 @@ def get_platform_status(self): :class:`APIError` if the request was unsuccessful ''' endpoint = voxu.urljoin(self.base_url, "status", "all") - res = self._requests.get(endpoint, headers=self._header) - _validate_response(res) + res = self._request_get(endpoint, headers=self._header) return _parse_json_response(res)["statuses"] # PRIVATE METHODS ######################################################### @@ -1224,6 +1209,11 @@ def _stream_download(self, url, output_path): for chunk in res.iter_content(chunk_size=_CHUNK_SIZE): f.write(chunk) + @retry(wait_exponential_multiplier=100, 10000) + def _request_get(self, *args, **kwargs): + res = self._requests.get(*args, **kwargs) + _validate_response(res) + return res class APIError(Exception): '''Exception raised when an :class:`API` request fails.''' From f316ddd462bc393aa68305cf942ebc808aac056c Mon Sep 17 00:00:00 2001 From: Tyler Ganter Date: Wed, 8 Apr 2020 10:27:24 -0400 Subject: [PATCH 2/3] bug --- voxel51/users/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/voxel51/users/api.py b/voxel51/users/api.py index 4cfa45f..0d19a16 100644 --- a/voxel51/users/api.py +++ b/voxel51/users/api.py @@ -1209,7 +1209,7 @@ def _stream_download(self, url, output_path): for chunk in res.iter_content(chunk_size=_CHUNK_SIZE): f.write(chunk) - @retry(wait_exponential_multiplier=100, 10000) + @retry(wait_exponential_multiplier=100, wait_exponential_max=10000) def _request_get(self, *args, **kwargs): res = self._requests.get(*args, **kwargs) _validate_response(res) From a2eb51173e8112b91ed3d3c040ac8ce901112257 Mon Sep 17 00:00:00 2001 From: Tyler Ganter Date: Wed, 8 Apr 2020 12:11:56 -0400 Subject: [PATCH 3/3] req --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2af92e1..7cccd40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ python-dateutil==2.7.0 pytz==2019.3 requests==2.20.0 requests-toolbelt==0.9.1 +retrying==1.3.3 six==1.12.0 Sphinx==1.7.5 sphinx-rtd-theme==0.2.4