diff --git a/minio/api.py b/minio/api.py index 38d208816..8e8be8e7e 100644 --- a/minio/api.py +++ b/minio/api.py @@ -72,7 +72,7 @@ get_sha256_hexdigest, get_md5_base64digest, Hasher, optimal_part_info, is_valid_bucket_name, PartMetadata, read_full, - get_s3_region_from_endpoint, is_valid_sse_object, is_valid_sse_c_object, + get_s3_region_from_endpoint, is_valid_sse_object, is_valid_sse_c_object, is_valid_source_sse_object, is_valid_bucket_notification_config, is_valid_policy_type, get_s3_region_from_endpoint, @@ -496,6 +496,14 @@ def listen_bucket_notification(self, bucket_name, prefix='', suffix='', """ is_valid_bucket_name(bucket_name) + # If someone explicitly set prefix to None convert it to empty string. + if prefix is None: + prefix = '' + + # If someone explicitly set suffix to None convert it to empty string. + if suffix is None: + suffix = '' + url_components = urlsplit(self._endpoint_url) if url_components.hostname == 's3.amazonaws.com': raise InvalidArgumentError( @@ -697,7 +705,7 @@ def copy_object(self, bucket_name, object_name, object_source, :param object_source: Source object to be copied. :param conditions: :class:`CopyConditions` object. Collection of supported CopyObject conditions. - :param metadata: Any user-defined metadata to be copied along with + :param metadata: Any user-defined metadata to be copied along with destination object. """ is_valid_bucket_name(bucket_name) @@ -710,16 +718,16 @@ def copy_object(self, bucket_name, object_name, object_source, if metadata is not None: headers = amzprefix_user_metadata(metadata) headers["x-amz-metadata-directive"] = "REPLACE" - + if conditions: for k, v in conditions.items(): - headers[k] = v - + headers[k] = v + # Source argument to copy_object can only be of type copy_SSE_C if source_sse: is_valid_source_sse_object(source_sse) headers.update(source_sse.marshal()) - + #Destination argument to copy_object cannot be of type copy_SSE_C if sse: is_valid_sse_object(sse) @@ -732,10 +740,10 @@ def copy_object(self, bucket_name, object_name, object_source, headers=headers) return parse_copy_object(bucket_name, object_name, response.data) - + def put_object(self, bucket_name, object_name, data, length, content_type='application/octet-stream', - metadata=None, + metadata=None, sse=None, ): """ @@ -796,7 +804,7 @@ def put_object(self, bucket_name, object_name, data, length, metadata=metadata, sse=sse) - def list_objects(self, bucket_name, prefix=None, recursive=False): + def list_objects(self, bucket_name, prefix='', recursive=False): """ List objects in the given bucket. @@ -835,17 +843,18 @@ def list_objects(self, bucket_name, prefix=None, recursive=False): """ is_valid_bucket_name(bucket_name) + # If someone explicitly set prefix to None convert it to empty string. + if prefix is None: + prefix = '' + method = 'GET' # Initialize query parameters. query = { - 'max-keys': '1000' + 'max-keys': '1000', + 'prefix': prefix } - # Add if prefix present. - if prefix: - query['prefix'] = prefix - # Delimited by default. if not recursive: query['delimiter'] = '/' @@ -865,7 +874,7 @@ def list_objects(self, bucket_name, prefix=None, recursive=False): for obj in objects: yield obj - def list_objects_v2(self, bucket_name, prefix=None, recursive=False): + def list_objects_v2(self, bucket_name, prefix='', recursive=False): """ List objects in the given bucket using the List objects V2 API. @@ -904,13 +913,15 @@ def list_objects_v2(self, bucket_name, prefix=None, recursive=False): """ is_valid_bucket_name(bucket_name) + # If someone explicitly set prefix to None convert it to empty string. + if prefix is None: + prefix = '' + # Initialize query parameters. query = { - 'list-type': '2' + 'list-type': '2', + 'prefix': prefix } - # Add if prefix present. - if prefix: - query['prefix'] = prefix # Delimited by default. if not recursive: @@ -1058,7 +1069,7 @@ def remove_objects(self, bucket_name, objects_iter): # clear batch for next set of items obj_batch = [] - def list_incomplete_uploads(self, bucket_name, prefix=None, + def list_incomplete_uploads(self, bucket_name, prefix='', recursive=False): """ List all in-complete uploads for a given bucket. @@ -1104,7 +1115,7 @@ def list_incomplete_uploads(self, bucket_name, prefix=None, return self._list_incomplete_uploads(bucket_name, prefix, recursive) - def _list_incomplete_uploads(self, bucket_name, prefix=None, + def _list_incomplete_uploads(self, bucket_name, prefix='', recursive=False, is_aggregate_size=True): """ List incomplete uploads list all previously uploaded incomplete multipart objects. @@ -1116,18 +1127,21 @@ def _list_incomplete_uploads(self, bucket_name, prefix=None, """ is_valid_bucket_name(bucket_name) + # If someone explicitly set prefix to None convert it to empty string. + if prefix is None: + prefix = '' + # Initialize query parameters. query = { 'uploads': '', - 'max-uploads': '1000' + 'max-uploads': '1000', + 'prefix': prefix } - if prefix: - query['prefix'] = prefix if not recursive: query['delimiter'] = '/' - key_marker, upload_id_marker = None, None + key_marker, upload_id_marker = '', '' is_truncated = True while is_truncated: if key_marker: @@ -1181,7 +1195,7 @@ def _list_object_parts(self, bucket_name, object_name, upload_id): } is_truncated = True - part_number_marker = None + part_number_marker = '' while is_truncated: if part_number_marker: query['part-number-marker'] = str(part_number_marker) @@ -1423,7 +1437,7 @@ def _get_partial_object(self, bucket_name, object_name, if sse: headers.update(sse.marshal()) - + return self._url_open('GET', bucket_name=bucket_name, object_name=object_name, @@ -1458,8 +1472,8 @@ def _do_put_object(self, bucket_name, object_name, part_data, 'Content-Length': part_size, } - md5_base64 = None - sha256_hex = None + md5_base64 = '' + sha256_hex = '' if self._is_ssl: md5_base64 = get_md5_base64digest(part_data) sha256_hex = _UNSIGNED_PAYLOAD diff --git a/minio/helpers.py b/minio/helpers.py index 83fba388e..6f3a6af3a 100644 --- a/minio/helpers.py +++ b/minio/helpers.py @@ -255,20 +255,14 @@ def get_target_url(endpoint_url, bucket_name=None, object_name=None, ordered_query = collections.OrderedDict(sorted(query.items())) query_components = [] for component_key in ordered_query: - if ordered_query[component_key] is not None: - if isinstance(ordered_query[component_key], list): - for value in ordered_query[component_key]: - query_components.append(component_key+'='+ - queryencode(value)) - else: - query_components.append( - component_key+'='+ - queryencode( - ordered_query[component_key] - ) - ) + if isinstance(ordered_query[component_key], list): + for value in ordered_query[component_key]: + query_components.append(component_key+'='+ + queryencode(value)) else: - query_components.append(component_key) + query_components.append( + component_key+'='+ + queryencode(ordered_query.get(component_key, ''))) query_string = '&'.join(query_components) if query_string: diff --git a/minio/signer.py b/minio/signer.py index 1d15b29cb..ad66968a6 100644 --- a/minio/signer.py +++ b/minio/signer.py @@ -125,6 +125,8 @@ def presign_v4(method, url, access_key, secret_key, region=None, single_component.append( queryencode(ordered_query[component_key]) ) + else: + single_component.append('=') query_components.append(''.join(single_component)) query_string = '&'.join(query_components) diff --git a/tests/functional/tests.py b/tests/functional/tests.py old mode 100755 new mode 100644 index e02d661e5..d77f2b5b9 --- a/tests/functional/tests.py +++ b/tests/functional/tests.py @@ -392,7 +392,7 @@ def test_copy_object_with_metadata(client, log_output): KB_1 = 1024 # 1KiB. KB_1_reader = LimitedRandomReader(KB_1) client.put_object(bucket_name, object_source, KB_1_reader, KB_1) - + # Perform a server side copy of an object client.copy_object(bucket_name, object_copy, '/'+bucket_name+'/'+object_source,metadata=metadata) @@ -411,7 +411,7 @@ def test_copy_object_with_metadata(client, log_output): raise Exception(err) # Test passes print(log_output.json_report()) - + def test_copy_object_etag_match(client, log_output): # default value for log_output.function attribute is; # log_output.function = "copy_object(bucket_name, object_name, object_source, conditions)" @@ -578,7 +578,7 @@ def normalize_metadata(meta_data): norm_dict = {k.lower(): v for k, v in meta_data.items()} return norm_dict - + def test_put_object(client, log_output, sse=None): # default value for log_output.function attribute is; # log_output.function = "put_object(bucket_name, object_name, data, length, content_type, metadata)" @@ -907,7 +907,7 @@ def test_list_objects(client, log_output): client.put_object(bucket_name, object_name+"-2", MB_1_reader, MB_1) # List all object paths in bucket. log_output.args['recursive'] = is_recursive = True - objects = client.list_objects(bucket_name, None, is_recursive) + objects = client.list_objects(bucket_name, '', is_recursive) for obj in objects: _, _, _, _, _, _ = obj.bucket_name,\ obj.object_name,\ @@ -1081,7 +1081,7 @@ def test_list_objects_v2(client, log_output): client.put_object(bucket_name, object_name+"-2", MB_1_reader, MB_1) # List all object paths in bucket using V2 API. log_output.args['recursive'] = is_recursive = True - objects = client.list_objects_v2(bucket_name, None, is_recursive) + objects = client.list_objects_v2(bucket_name, '', is_recursive) for obj in objects: _, _, _, _, _, _ = obj.bucket_name,\ obj.object_name,\ diff --git a/tests/unit/list_incomplete_uploads_test.py b/tests/unit/list_incomplete_uploads_test.py index 4d64e3120..cdd9c6387 100644 --- a/tests/unit/list_incomplete_uploads_test.py +++ b/tests/unit/list_incomplete_uploads_test.py @@ -45,7 +45,7 @@ def test_empty_list_uploads_test(self, mock_connection): mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('GET', - 'https://localhost:9000/bucket/?max-uploads=1000&uploads=', + 'https://localhost:9000/bucket/?max-uploads=1000&prefix=&uploads=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') upload_iter = client._list_incomplete_uploads('bucket', '', True, False) @@ -102,7 +102,7 @@ def test_list_uploads_works(self, mock_connection): mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('GET', - 'https://localhost:9000/bucket/?delimiter=%2F&max-uploads=1000&uploads=', + 'https://localhost:9000/bucket/?delimiter=%2F&max-uploads=1000&prefix=&uploads=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) @@ -203,7 +203,7 @@ def test_list_multipart_uploads_works(self, mock_connection): mock_connection.return_value = mock_server mock_server.mock_add_request( MockResponse('GET', - 'https://localhost:9000/bucket/?max-uploads=1000&uploads=', + 'https://localhost:9000/bucket/?max-uploads=1000&prefix=&uploads=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data1)) client = Minio('localhost:9000') @@ -213,7 +213,7 @@ def test_list_multipart_uploads_works(self, mock_connection): mock_server.mock_add_request(MockResponse('GET', 'https://localhost:9000/bucket/?' 'key-marker=keymarker&' - 'max-uploads=1000&' + 'max-uploads=1000&prefix=&' 'upload-id-marker=uploadidmarker&uploads=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data2)) uploads.append(upload) diff --git a/tests/unit/list_objects_test.py b/tests/unit/list_objects_test.py index 574507507..958cd28d2 100644 --- a/tests/unit/list_objects_test.py +++ b/tests/unit/list_objects_test.py @@ -40,7 +40,7 @@ def test_empty_list_objects_works(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', - 'https://localhost:9000/bucket/?max-keys=1000', + 'https://localhost:9000/bucket/?max-keys=1000&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') bucket_iter = client.list_objects('bucket', recursive=True) @@ -87,7 +87,7 @@ def test_list_objects_works(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', - 'https://localhost:9000/bucket/?delimiter=%2F&max-keys=1000', + 'https://localhost:9000/bucket/?delimiter=%2F&max-keys=1000&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') bucket_iter = client.list_objects('bucket') @@ -95,7 +95,7 @@ def test_list_objects_works(self, mock_connection): for bucket in bucket_iter: # cause an xml exception and fail if we try retrieving again mock_server.mock_add_request(MockResponse('GET', - 'https://localhost:9000/bucket/?delimiter=%2F&max-keys=1000', + 'https://localhost:9000/bucket/?delimiter=%2F&max-keys=1000&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content='')) buckets.append(bucket) @@ -172,13 +172,13 @@ def test_list_objects_works_well(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', - 'https://localhost:9000/bucket/?max-keys=1000', + 'https://localhost:9000/bucket/?max-keys=1000&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data1)) client = Minio('localhost:9000') bucket_iter = client.list_objects('bucket', recursive=True) buckets = [] for bucket in bucket_iter: - url = 'https://localhost:9000/bucket/?marker=marker&max-keys=1000' + url = 'https://localhost:9000/bucket/?marker=marker&max-keys=1000&prefix=' mock_server.mock_add_request(MockResponse('GET', url, {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data2)) diff --git a/tests/unit/list_objects_v2_test.py b/tests/unit/list_objects_v2_test.py index d8b05f901..4f37ad8bf 100644 --- a/tests/unit/list_objects_v2_test.py +++ b/tests/unit/list_objects_v2_test.py @@ -39,7 +39,7 @@ def test_empty_list_objects_works(self, mock_connection): mock_server = MockConnection() mock_connection.return_value = mock_server mock_server.mock_add_request(MockResponse('GET', - 'https://localhost:9000/bucket/?list-type=2', + 'https://localhost:9000/bucket/?list-type=2&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data)) client = Minio('localhost:9000') object_iter = client.list_objects_v2('bucket', recursive=True) @@ -79,7 +79,7 @@ def test_list_objects_works(self, mock_connection): mock_server.mock_add_request( MockResponse( 'GET', - 'https://localhost:9000/bucket/?delimiter=%2F&list-type=2', + 'https://localhost:9000/bucket/?delimiter=%2F&list-type=2&prefix=', {'User-Agent': _DEFAULT_USER_AGENT}, 200, content=mock_data )