From abe00cf45423476e96da47e794bbdbddfe75272e Mon Sep 17 00:00:00 2001 From: Andrew McGrath Date: Tue, 9 Jun 2020 13:32:48 -0400 Subject: [PATCH] This commit moves the regions.py vendored module into chalice/vendored/botocore and removes type checking, docstring and other validations for this vendored file in order to use the file without alteration. --- .pylintrc | 2 +- Makefile | 4 +- chalice/awsclient.py | 2 +- chalice/vendored/__init__.py | 0 chalice/vendored/botocore/__init__.py | 0 chalice/{ => vendored/botocore}/regions.py | 138 ++++++++---------- setup.cfg | 3 + tests/unit/vendored/__init__.py | 0 tests/unit/vendored/botocore/__init__.py | 0 .../{ => vendored/botocore}/test_regions.py | 2 +- 10 files changed, 70 insertions(+), 81 deletions(-) create mode 100644 chalice/vendored/__init__.py create mode 100644 chalice/vendored/botocore/__init__.py rename chalice/{ => vendored/botocore}/regions.py (77%) create mode 100644 tests/unit/vendored/__init__.py create mode 100644 tests/unit/vendored/botocore/__init__.py rename tests/unit/{ => vendored/botocore}/test_regions.py (99%) diff --git a/.pylintrc b/.pylintrc index 4004c4b848..6e016ad44f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -9,7 +9,7 @@ # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=compat.py +ignore=compat.py,regions.py # Pickle collected data for later comparisons. persistent=yes diff --git a/Makefile b/Makefile index 087c051f16..141065bc15 100644 --- a/Makefile +++ b/Makefile @@ -6,14 +6,14 @@ TESTS=tests/unit tests/functional tests/integration check: ###### FLAKE8 ##### # No unused imports, no undefined vars, - flake8 --ignore=E731,W503,W504 --exclude chalice/__init__.py,chalice/compat.py --max-complexity 10 chalice/ + flake8 --ignore=E731,W503,W504 --exclude chalice/__init__.py,chalice/compat.py,chalice/vendored/botocore/regions.py --max-complexity 10 chalice/ flake8 --ignore=E731,W503,W504,F401 --max-complexity 10 chalice/compat.py flake8 tests/unit/ tests/functional/ tests/integration tests/aws # # Proper docstring conventions according to pep257 # # - pydocstyle --add-ignore=D100,D101,D102,D103,D104,D105,D204,D301 chalice/ + pydocstyle --add-ignore=D100,D101,D102,D103,D104,D105,D204,D301 --match='(?!(test_|regions)).*\.py' chalice/ pylint: ###### PYLINT ###### diff --git a/chalice/awsclient.py b/chalice/awsclient.py index d7b27a09d1..529085de01 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -37,7 +37,7 @@ from chalice.constants import DEFAULT_STAGE_NAME from chalice.constants import MAX_LAMBDA_DEPLOYMENT_SIZE -from chalice.regions import EndpointResolver +from chalice.vendored.botocore.regions import EndpointResolver StrMap = Optional[Dict[str, str]] OptStr = Optional[str] diff --git a/chalice/vendored/__init__.py b/chalice/vendored/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/chalice/vendored/botocore/__init__.py b/chalice/vendored/botocore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/chalice/regions.py b/chalice/vendored/botocore/regions.py similarity index 77% rename from chalice/regions.py rename to chalice/vendored/botocore/regions.py index 6385aedca5..802f90b03d 100644 --- a/chalice/regions.py +++ b/chalice/vendored/botocore/regions.py @@ -15,55 +15,58 @@ This module implements endpoint resolution, including resolving endpoints for a given service and region and resolving the available endpoints for a service in a specific AWS partition. - -This file was 'vendored' from botocore core botocore/botocore/regions.py from -commit 0c55d6c3f900fc856e818f06b31c22c6dbc56788. The vendoring/duplication -was due to the concern of utilizing a unexposed class internal to the botocore -library for functionality necessary to implicitly support partitions within -the chalice microframework. More specifically the determination of the dns -suffix for service endpoints based on service and region. - -https://github.com/boto/botocore/tree/0c55d6c3f900fc856e818f06b31c22c6dbc56788 """ import logging import re -from typing import List, Dict, Any # noqa from botocore.exceptions import NoRegionError -logger = logging.getLogger(__name__) +LOG = logging.getLogger(__name__) DEFAULT_URI_TEMPLATE = '{service}.{region}.{dnsSuffix}' DEFAULT_SERVICE_DATA = {'endpoints': {}} -class EndpointResolver(object): - """Resolve endpoints based on partition endpoint metadata.""" +class BaseEndpointResolver(object): + """Resolves regions and endpoints. Must be subclassed.""" + def construct_endpoint(self, service_name, region_name=None): + """Resolves an endpoint for a service and region combination. - def __init__(self, endpoint_data): - # type: (Dict[str, Any]) -> None - """Construct a new EndpointResolver based on a parsed endpoints.json. + :type service_name: string + :param service_name: Name of the service to resolve an endpoint for + (e.g., s3) - :param endpoint_data: A dict of partition data. + :type region_name: string + :param region_name: Region/endpoint name to resolve (e.g., us-east-1) + if no region is provided, the first found partition-wide endpoint + will be used if available. + + :rtype: dict + :return: Returns a dict containing the following keys: + - partition: (string, required) Resolved partition name + - endpointName: (string, required) Resolved endpoint name + - hostname: (string, required) Hostname to use for this endpoint + - sslCommonName: (string) sslCommonName to use for this endpoint. + - credentialScope: (dict) Signature version 4 credential scope + - region: (string) region name override when signing. + - service: (string) service name override when signing. + - signatureVersions: (list) A list of possible signature + versions, including s3, v4, v2, and s3v4 + - protocols: (list) A list of supported protocols + (e.g., http, https) + - ...: Other keys may be included as well based on the metadata """ - if 'partitions' not in endpoint_data: - raise ValueError('Missing "partitions" in endpoint data') - self._endpoint_data = endpoint_data + raise NotImplementedError def get_available_partitions(self): - # type: () -> List[str] - """List the partitions available to the endpoint resolver. + """Lists the partitions available to the endpoint resolver. :return: Returns a list of partition names (e.g., ["aws", "aws-cn"]). """ - result = [] - for partition in self._endpoint_data['partitions']: - result.append(partition['partition']) - return result + raise NotImplementedError def get_available_endpoints(self, service_name, partition_name='aws', allow_non_regional=False): - # type: (str, str, bool) -> List[str] - """List the endpoint names of a particular partition. + """Lists the endpoint names of a particular partition. :type service_name: string :param service_name: Name of a service to list endpoint for (e.g., s3) @@ -79,6 +82,27 @@ def get_available_endpoints(self, service_name, partition_name='aws', fips-us-gov-west-1, etc). :return: Returns a list of endpoint names (e.g., ["us-east-1"]). """ + raise NotImplementedError + + +class EndpointResolver(BaseEndpointResolver): + """Resolves endpoints based on partition endpoint metadata""" + def __init__(self, endpoint_data): + """ + :param endpoint_data: A dict of partition data. + """ + if 'partitions' not in endpoint_data: + raise ValueError('Missing "partitions" in endpoint data') + self._endpoint_data = endpoint_data + + def get_available_partitions(self): + result = [] + for partition in self._endpoint_data['partitions']: + result.append(partition['partition']) + return result + + def get_available_endpoints(self, service_name, partition_name='aws', + allow_non_regional=False): result = [] for partition in self._endpoint_data['partitions']: if partition['partition'] != partition_name: @@ -91,40 +115,7 @@ def get_available_endpoints(self, service_name, partition_name='aws', result.append(endpoint_name) return result - def construct_endpoint(self, service_name, region_name=None, - partition_name=None): - # type: (str, str, str) -> Dict[str, Any] - """Resolve an endpoint for a service and region combination. - - :type service_name: string - :param service_name: Name of the service to resolve an endpoint for - (e.g., s3) - - :type region_name: string - :param region_name: Region/endpoint name to resolve (e.g., us-east-1) - if no region is provided, the first found partition-wide endpoint - will be used if available. - - :type partition_name: string - :param partition_name: Partition name to resolve (e.g., aws, aws-cn) - if no partition is provided, the first found partition-wide - endpoint will be used if available. - - :rtype: dict - :return: Returns a dict containing the following keys: - - partition: (string, required) Resolved partition name - - endpointName: (string, required) Resolved endpoint name - - hostname: (string, required) Hostname to use for this endpoint - - sslCommonName: (string) sslCommonName to use for this endpoint. - - credentialScope: (dict) Signature version 4 credential scope - - region: (string) region name override when signing. - - service: (string) service name override when signing. - - signatureVersions: (list) A list of possible signature - versions, including s3, v4, v2, and s3v4 - - protocols: (list) A list of supported protocols - (e.g., http, https) - - ...: Other keys may be included as well based on the metadata - """ + def construct_endpoint(self, service_name, region_name=None, partition_name=None): if partition_name is not None: valid_partition = None for partition in self._endpoint_data['partitions']: @@ -132,9 +123,8 @@ def construct_endpoint(self, service_name, region_name=None, valid_partition = partition if valid_partition is not None: - result = self._endpoint_for_partition(valid_partition, - service_name, - region_name, True) + result = self._endpoint_for_partition(valid_partition, service_name, + region_name, True) return result return None @@ -146,8 +136,7 @@ def construct_endpoint(self, service_name, region_name=None, return result def _endpoint_for_partition(self, partition, service_name, region_name, - force_partition=False): - # type: (str, str, str, bool) -> Dict[str, Any] + force_partition=False): # Get the service from the partition, or an empty template. service_data = partition['services'].get( service_name, DEFAULT_SERVICE_DATA) @@ -167,17 +156,16 @@ def _endpoint_for_partition(self, partition, service_name, region_name, partition_endpoint = service_data.get('partitionEndpoint') is_regionalized = service_data.get('isRegionalized', True) if partition_endpoint and not is_regionalized: - logger.debug('Using partition endpoint for %s, %s: %s', - service_name, region_name, partition_endpoint) + LOG.debug('Using partition endpoint for %s, %s: %s', + service_name, region_name, partition_endpoint) return self._resolve( partition, service_name, service_data, partition_endpoint) - logger.debug('Creating a regex based endpoint for %s, %s', - service_name, region_name) + LOG.debug('Creating a regex based endpoint for %s, %s', + service_name, region_name) return self._resolve( partition, service_name, service_data, region_name) def _region_match(self, partition, region_name): - # type: (str, str) -> bool if region_name in partition['regions']: return True if 'regionRegex' in partition: @@ -185,13 +173,13 @@ def _region_match(self, partition, region_name): return False def _resolve(self, partition, service_name, service_data, endpoint_name): - # type: (str, str, str, str) -> Dict[str, Any] result = service_data['endpoints'].get(endpoint_name, {}) result['partition'] = partition['partition'] result['endpointName'] = endpoint_name # Merge in the service defaults then the partition defaults. self._merge_keys(service_data.get('defaults', {}), result) self._merge_keys(partition.get('defaults', {}), result) + hostname = result.get('hostname', DEFAULT_URI_TEMPLATE) result['hostname'] = self._expand_template( partition, result['hostname'], service_name, endpoint_name) if 'sslCommonName' in result: @@ -202,14 +190,12 @@ def _resolve(self, partition, service_name, service_data, endpoint_name): return result def _merge_keys(self, from_data, result): - # type: (Dict[str, Any], Dict[str, Any]) -> None for key in from_data: if key not in result: result[key] = from_data[key] def _expand_template(self, partition, template, service_name, endpoint_name): - # type: (Dict[str, Any], str, str, str) -> str return template.format( service=service_name, region=endpoint_name, - dnsSuffix=partition['dnsSuffix']) + dnsSuffix=partition['dnsSuffix']) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 5e4090017a..e77fab171b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [wheel] universal = 1 + +[mypy-chalice.vendored.*] +ignore_errors = true \ No newline at end of file diff --git a/tests/unit/vendored/__init__.py b/tests/unit/vendored/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/vendored/botocore/__init__.py b/tests/unit/vendored/botocore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/test_regions.py b/tests/unit/vendored/botocore/test_regions.py similarity index 99% rename from tests/unit/test_regions.py rename to tests/unit/vendored/botocore/test_regions.py index de8e747979..2b17347308 100644 --- a/tests/unit/test_regions.py +++ b/tests/unit/vendored/botocore/test_regions.py @@ -26,7 +26,7 @@ import pytest from botocore.exceptions import NoRegionError -from chalice import regions +from chalice.vendored.botocore import regions @pytest.fixture