From 1d97393367490c5e3ef331d4031a3ff88de2e573 Mon Sep 17 00:00:00 2001 From: Wouter Toering Date: Fri, 14 Feb 2020 15:37:21 +0100 Subject: [PATCH 1/4] Restore upload functionality --- bynder_sdk/client/upload_client.py | 79 +++++++++++++----------------- 1 file changed, 34 insertions(+), 45 deletions(-) diff --git a/bynder_sdk/client/upload_client.py b/bynder_sdk/client/upload_client.py index 13d3e2e..ef5f54d 100644 --- a/bynder_sdk/client/upload_client.py +++ b/bynder_sdk/client/upload_client.py @@ -1,3 +1,4 @@ +import math import os import time @@ -14,10 +15,6 @@ class UploadClient(): def __init__(self, session): self.session = session - @staticmethod - def _retrieve_filename(file_path): - return file_path.rsplit('/', 1)[-1] - def upload(self, file_path, media_id, upload_data): """ Handles the upload of the file. """ @@ -26,45 +23,34 @@ def upload(self, file_path, media_id, upload_data): return self._save_media(finalise_data['importId'], upload_data, media_id) - @staticmethod - def _update_multipart(filename, total_parts, init_data, part_nr): - key = init_data['multipart_params']['key'] - init_data['multipart_params'].update({ - 'name': filename, - 'chunk': part_nr, - 'chunks': total_parts, - 'Filename': key - }) - return init_data - def _run_s3_upload(self, file_path): """ Uploads the media to Amazon S3 bucket-endpoint. """ - filename = self._retrieve_filename(file_path) - init_data = self._init_upload(filename) with open(file_path, 'rb') as f: part_nr = 0 - total_parts = (os.fstat(f.fileno()).st_size + MAX_CHUNK_SIZE - 1)\ - // MAX_CHUNK_SIZE + total_parts = math.ceil( + os.stat(f.fileno()).st_size / MAX_CHUNK_SIZE) + + filename = file_path.rsplit('/', 1)[-1] + build_part_data = self._init_upload(filename, total_parts) part_bytes = f.read(MAX_CHUNK_SIZE) while part_bytes: part_nr = part_nr + 1 - init_data = self._update_init_data(init_data, part_nr) - init_data = self._update_multipart(filename, total_parts, - init_data, part_nr) + part_data = build_part_data(part_nr) + response = self.session.post( self.upload_url, - files={"file": part_bytes}, - data=init_data['multipart_params'], - withhold_token=True, + files={'file': part_bytes}, + data=part_data['multipart_params'], ) response.raise_for_status() - self._register_part(init_data, part_nr) + self._register_part(part_data, part_nr) part_bytes = f.read(MAX_CHUNK_SIZE) - return init_data, total_parts - def _init_upload(self, filename): + return part_data, total_parts + + def _init_upload(self, filename, total_parts): """ Gets the URL of the Amazon S3 bucket-endpoint in the region closest to the server and initialises a file upload with Bynder and returns authorisation information to allow uploading to the @@ -72,23 +58,27 @@ def _init_upload(self, filename): """ self.upload_url = self.session.get('/upload/endpoint/') - payload = {'filename': filename} - return self.session.post( + data = self.session.post( '/upload/init/', - data=payload + data={'filename': filename} ) + data['multipart_params'].update({ + 'chunks': total_parts, + 'name': filename, + }) - @staticmethod - def _update_init_data(init_data, part_nr): - """ Updates the init data. - """ - key = '{}/p{}'.format( - init_data['s3_filename'], - part_nr - ) - init_data['s3_filename'] = key - init_data['multipart_params']['key'] = key - return init_data + key = '{}/p{{}}'.format(data['s3_filename']) + + def part_data(part_nr): + data['s3_filename'] = key.format(part_nr) + data['multipart_params'].update({ + 'chunk': part_nr, + 'Filename': key.format(part_nr), + 'key': key.format(part_nr), + }) + return data + + return part_data def _register_part(self, init_data, part_nr): """ Registers an uploaded chunk in Bynder. @@ -107,8 +97,7 @@ def _finalise_file(self, init_data, total_parts): """ Finalises a completely uploaded file. """ return self.session.post( - '/v4/upload/{0}/'.format( - init_data['s3file']['uploadid']), + '/v4/upload/{0}/'.format(init_data['s3file']['uploadid']), data={ 'id': init_data['s3file']['uploadid'], 'targetid': init_data['s3file']['targetid'], @@ -124,7 +113,7 @@ def _save_media(self, import_id, data, media_id=None): """ poll_status = self._poll_status(import_id) if import_id not in poll_status['itemsDone']: - raise Exception("Converting media failed") + raise Exception('Converting media failed') save_endpoint = '/v4/media/save/{}/'.format(import_id) if media_id: From b74a906198d2727be0399a14d582e608fe433aac Mon Sep 17 00:00:00 2001 From: Wouter Toering Date: Fri, 14 Feb 2020 15:37:50 +0100 Subject: [PATCH 2/4] Reduce polling idle time, idle before polling --- bynder_sdk/client/upload_client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bynder_sdk/client/upload_client.py b/bynder_sdk/client/upload_client.py index ef5f54d..7a8b328 100644 --- a/bynder_sdk/client/upload_client.py +++ b/bynder_sdk/client/upload_client.py @@ -5,7 +5,7 @@ MAX_CHUNK_SIZE = 1024 * 1024 * 5 MAX_POLLING_ITERATIONS = 60 -POLLING_IDLE_TIME = 10 +POLLING_IDLE_TIME = 5 # pylint: disable-msg=too-few-public-methods @@ -130,6 +130,7 @@ def _poll_status(self, import_id): """ Gets poll processing status of finalised files. """ for _ in range(MAX_POLLING_ITERATIONS): + time.sleep(POLLING_IDLE_TIME) status_dict = self.session.get( '/v4/upload/poll/', params={'items': [import_id]} @@ -138,8 +139,6 @@ def _poll_status(self, import_id): if [v for k, v in status_dict.items() if import_id in v]: return status_dict - time.sleep(POLLING_IDLE_TIME) - # Max polling iterations reached => upload failed status_dict['itemsFailed'].append(import_id) return status_dict From 19c6b9f4df48dbb95c47bacf38f5ce5ba2c0ec1e Mon Sep 17 00:00:00 2001 From: Wouter Toering Date: Wed, 19 Feb 2020 11:18:44 +0100 Subject: [PATCH 3/4] Ensure auth header isn't sent to S3 This results in rejected uploads --- bynder_sdk/util.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bynder_sdk/util.py b/bynder_sdk/util.py index a7f6cfb..4e38a88 100644 --- a/bynder_sdk/util.py +++ b/bynder_sdk/util.py @@ -23,6 +23,8 @@ def get(self, url, *args, **kwargs): def post(self, url, *args, **kwargs): if url.startswith('https'): + # Do not send the Authorization header to S3 + kwargs['headers'] = {'Authorization': None} return super().post(url, *args, **kwargs) return self.wrapped_request(super().post, url, *args, **kwargs) From 109d5ecc31dd921f4599c9fe3f17d846e8d73f7a Mon Sep 17 00:00:00 2001 From: Wouter Toering Date: Wed, 19 Feb 2020 11:38:02 +0100 Subject: [PATCH 4/4] Bump version to 1.1.2 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 524cb55..45a1b3f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.1 +1.1.2