Merge branch 'release-1.9.0'

* release-1.9.0:
  Bumping version to 1.9.0
  Update to latest models
  Added changelog for metadata stubbing
  support setting response metadata when adding client error to stub
  Add changelog entry for #1387
  Use regional endpoint for dualstack endpoints
  Use global endpoint for s3 presigning
  Account for 400 response from HeadBucket/HeadObject
  Remove special case govcloud handling
  Remove hardcoded 's3.amazonaws.com' for virtual hosted addressing
  Decouple signature version of S3 addressing style

.changes/1.9.0.json

@@ -0,0 +1,22 @@
+[
+  {
+    "category": "Stubber", 
+    "description": "Added the ability to add items to response metadata with the stubber.", 
+    "type": "enhancement"
+  }, 
+  {
+    "category": "``sts``", 
+    "description": "Update sts client to latest version", 
+    "type": "api-change"
+  }, 
+  {
+    "category": "``route53``", 
+    "description": "Update route53 client to latest version", 
+    "type": "api-change"
+  }, 
+  {
+    "category": "``s3``", 
+    "description": "Default to virtual hosted addressing regardless of signature version (boto/botocore`#1387 <https://github.com/boto/botocore/issues/1387>`__)", 
+    "type": "feature"
+  }
+]

CHANGELOG.rst

@@ -2,6 +2,15 @@
 CHANGELOG
 =========
 
+1.9.0
+=====
+
+* enhancement:Stubber: Added the ability to add items to response metadata with the stubber.
+* api-change:``sts``: Update sts client to latest version
+* api-change:``route53``: Update route53 client to latest version
+* feature:``s3``: Default to virtual hosted addressing regardless of signature version (boto/botocore`#1387 <https://github.com/boto/botocore/issues/1387>`__)
+
+
 1.8.50
 ======
 

botocore/__init__.py

@@ -16,7 +16,7 @@
 import re
 import logging
 
-__version__ = '1.8.50'
+__version__ = '1.9.0'
 
 
 class NullHandler(logging.Handler):

botocore/client.py

@@ -189,16 +189,6 @@ def _get_s3_addressing_handler(self, endpoint_url, s3_config,
         logger.debug("Defaulting to S3 virtual host style addressing with "
                      "path style addressing fallback.")
 
-        # For dual stack mode, we need to clear the default endpoint url in
-        # order to use the existing netloc if the bucket is dns compatible.
-        # Also, the default_endpoint_url of 's3.amazonaws.com' only works
-        # if we're in the 'aws' partition.  Anywhere else we should
-        # just use the existing netloc.
-        if s3_config.get('use_dualstack_endpoint', False) or \
-                partition != 'aws':
-            return functools.partial(
-                fix_s3_host, default_endpoint_url=None)
-
         # By default, try to use virtual style with path fallback.
         return fix_s3_host
 

botocore/data/route53/2013-04-01/service-2.json

@@ -4241,6 +4241,7 @@
         "ap-southeast-2",
         "ap-northeast-1",
         "ap-northeast-2",
+        "ap-northeast-3",
         "sa-east-1",
         "cn-north-1",
         "cn-northwest-1",

botocore/signers.py

@@ -558,7 +558,8 @@ def generate_presigned_url(self, ClientMethod, Params=None, ExpiresIn=3600,
     expires_in = ExpiresIn
     http_method = HttpMethod
     context = {
-        'is_presign_request': True
+        'is_presign_request': True,
+        'use_global_endpoint': _should_use_global_endpoint(self),
     }
 
     request_signer = self._request_signer
@@ -686,7 +687,12 @@ def generate_presigned_post(self, Bucket, Key, Fields=None, Conditions=None,
 
     # Prepare the request dict by including the client's endpoint url.
     prepare_request_dict(
-        request_dict, endpoint_url=self.meta.endpoint_url)
+        request_dict, endpoint_url=self.meta.endpoint_url,
+        context={
+            'is_presign_request': True,
+            'use_global_endpoint': _should_use_global_endpoint(self),
+        },
+    )
 
     # Append that the bucket name to the list of conditions.
     conditions.append({'bucket': bucket})
@@ -704,3 +710,12 @@ def generate_presigned_post(self, Bucket, Key, Fields=None, Conditions=None,
     return post_presigner.generate_presigned_post(
         request_dict=request_dict, fields=fields, conditions=conditions,
         expires_in=expires_in)
+
+
+def _should_use_global_endpoint(client):
+    use_dualstack_endpoint = False
+    if client.meta.config.s3 is not None:
+        use_dualstack_endpoint = client.meta.config.s3.get(
+            'use_dualstack_endpoint', False)
+    return (client.meta.partition == 'aws' and
+            not use_dualstack_endpoint)

botocore/stub.py

@@ -250,7 +250,8 @@ def _add_response(self, method, service_response, expected_params):
 
     def add_client_error(self, method, service_error_code='',
                          service_message='', http_status_code=400,
-                         service_error_meta=None, expected_params=None):
+                         service_error_meta=None, expected_params=None,
+                         response_meta=None):
         """
         Adds a ``ClientError`` to the response queue.
 
@@ -278,6 +279,11 @@ def add_client_error(self, method, service_error_code='',
             any of the parameters differ a ``StubResponseError`` is thrown.
             You can use stub.ANY to indicate a particular parameter to ignore
             in validation.
+
+        :param response_meta: Additional keys to be added to the
+            response's ResponseMetadata
+        :type response_meta: dict
+
         """
         http_response = Response()
         http_response.status_code = http_status_code
@@ -296,6 +302,9 @@ def add_client_error(self, method, service_error_code='',
         if service_error_meta is not None:
             parsed_response['Error'].update(service_error_meta)
 
+        if response_meta is not None:
+            parsed_response['ResponseMetadata'].update(response_meta)
+
         operation_name = self.client.meta.method_to_api_mapping.get(method)
         # Note that we do not allow for expected_params while
         # adding errors into the queue yet.

botocore/utils.py

@@ -41,10 +41,6 @@
 # Based on rfc2986, section 2.3
 SAFE_CHARS = '-._~'
 LABEL_RE = re.compile(r'[a-z0-9][a-z0-9\-]*[a-z0-9]')
-RESTRICTED_REGIONS = [
-    'us-gov-west-1',
-    'fips-us-gov-west-1',
-]
 RETRYABLE_HTTP_ERRORS = (requests.Timeout, requests.ConnectionError)
 S3_ACCELERATE_WHITELIST = ['dualstack']
 
@@ -672,23 +668,18 @@ def check_dns_name(bucket_name):
 
 
 def fix_s3_host(request, signature_version, region_name,
-                default_endpoint_url='s3.amazonaws.com', **kwargs):
+                default_endpoint_url=None, **kwargs):
     """
     This handler looks at S3 requests just before they are signed.
     If there is a bucket name on the path (true for everything except
     ListAllBuckets) it checks to see if that bucket name conforms to
     the DNS naming conventions.  If it does, it alters the request to
     use ``virtual hosting`` style addressing rather than ``path-style``
-    addressing.  This allows us to avoid 301 redirects for all
-    bucket names that can be CNAME'd.
+    addressing.
+
     """
-    # By default we do not use virtual hosted style addressing when
-    # signed with signature version 4.
-    if signature_version is not botocore.UNSIGNED and \
-            's3v4' in signature_version:
-        return
-    elif not _allowed_region(region_name):
-        return
+    if request.context.get('use_global_endpoint', False):
+        default_endpoint_url = 's3.amazonaws.com'
     try:
         switch_to_virtual_host_style(
             request, signature_version, default_endpoint_url)
@@ -765,10 +756,6 @@ def _is_get_bucket_location_request(request):
     return request.url.endswith('?location')
 
 
-def _allowed_region(region_name):
-    return region_name not in RESTRICTED_REGIONS
-
-
 def instance_cache(func):
     """Method decorator for caching method calls to a single instance.
 
@@ -904,14 +891,21 @@ def redirect_from_error(self, request_dict, response, operation, **kwargs):
         error = response[1].get('Error', {})
         error_code = error.get('Code')
 
-        if error_code == '301':
-            # A raw 301 error might be returned for several reasons, but we
-            # only want to try to redirect it if it's a HeadObject or
-            # HeadBucket  because all other operations will return
-            # PermanentRedirect if region is incorrect.
-            if operation.name not in ['HeadObject', 'HeadBucket']:
-                return
-        elif error_code != 'PermanentRedirect':
+        # We have to account for 400 responses because
+        # if we sign a Head* request with the wrong region,
+        # we'll get a 400 Bad Request but we won't get a
+        # body saying it's an "AuthorizationHeaderMalformed".
+        is_special_head_object = (
+            error_code in ['301', '400'] and
+            operation.name in ['HeadObject', 'HeadBucket']
+        )
+        is_wrong_signing_region = (
+            error_code == 'AuthorizationHeaderMalformed' and
+            'Region' in error
+        )
+        is_permanent_redirect = error_code == 'PermanentRedirect'
+        if not any([is_special_head_object, is_wrong_signing_region,
+                    is_permanent_redirect]):
             return
 
         bucket = request_dict['context']['signing']['bucket']

docs/source/conf.py

@@ -52,9 +52,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.8.'
+version = '1.9'
 # The full version, including alpha/beta/rc tags.
-release = '1.8.50'
+release = '1.9.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.