diff --git a/.pylintrc b/.pylintrc index 4004c4b848..6e016ad44f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -9,7 +9,7 @@ # Add files or directories to the blacklist. They should be base names, not # paths. -ignore=compat.py +ignore=compat.py,regions.py # Pickle collected data for later comparisons. persistent=yes diff --git a/Makefile b/Makefile index 087c051f16..141065bc15 100644 --- a/Makefile +++ b/Makefile @@ -6,14 +6,14 @@ TESTS=tests/unit tests/functional tests/integration check: ###### FLAKE8 ##### # No unused imports, no undefined vars, - flake8 --ignore=E731,W503,W504 --exclude chalice/__init__.py,chalice/compat.py --max-complexity 10 chalice/ + flake8 --ignore=E731,W503,W504 --exclude chalice/__init__.py,chalice/compat.py,chalice/vendored/botocore/regions.py --max-complexity 10 chalice/ flake8 --ignore=E731,W503,W504,F401 --max-complexity 10 chalice/compat.py flake8 tests/unit/ tests/functional/ tests/integration tests/aws # # Proper docstring conventions according to pep257 # # - pydocstyle --add-ignore=D100,D101,D102,D103,D104,D105,D204,D301 chalice/ + pydocstyle --add-ignore=D100,D101,D102,D103,D104,D105,D204,D301 --match='(?!(test_|regions)).*\.py' chalice/ pylint: ###### PYLINT ###### diff --git a/chalice/awsclient.py b/chalice/awsclient.py index edfb4ba1fd..a03d5ee913 100644 --- a/chalice/awsclient.py +++ b/chalice/awsclient.py @@ -13,31 +13,33 @@ this class to get improved type checking across chalice. """ +import json # pylint: disable=too-many-lines import os -import time -import tempfile -from datetime import datetime -import zipfile -import shutil -import json import re +import shutil +import tempfile +import time import uuid +import zipfile +from collections import OrderedDict +from datetime import datetime +from typing import Any, Optional, Dict, Callable, List, Iterator, Iterable, \ + IO, Union # noqa import botocore.session # noqa from botocore.exceptions import ClientError +from botocore.loaders import create_loader +from botocore.utils import datetime2timestamp from botocore.vendored.requests import ConnectionError as \ RequestsConnectionError from botocore.vendored.requests.exceptions import ReadTimeout as \ RequestsReadTimeout -from botocore.utils import datetime2timestamp -from typing import Any, Optional, Dict, Callable, List, Iterator, IO # noqa -from typing import Iterable # noqa from mypy_extensions import TypedDict from chalice.constants import DEFAULT_STAGE_NAME from chalice.constants import MAX_LAMBDA_DEPLOYMENT_SIZE - +from chalice.vendored.botocore.regions import EndpointResolver StrMap = Optional[Dict[str, str]] OptStr = Optional[str] @@ -61,7 +63,6 @@ }, total=False ) - _REMOTE_CALL_ERRORS = ( botocore.exceptions.ClientError, RequestsConnectionError ) @@ -95,9 +96,9 @@ class DeploymentPackageTooLargeError(LambdaClientError): class LambdaErrorContext(object): def __init__(self, - function_name, # type: str + function_name, # type: str client_method_name, # type: str - deployment_size, # type: int + deployment_size, # type: int ): # type: (...) -> None self.function_name = function_name @@ -106,7 +107,6 @@ def __init__(self, class TypedAWSClient(object): - # 30 * 5 == 150 seconds or 2.5 minutes for the initial lambda # creation + role propagation. LAMBDA_CREATE_ATTEMPTS = 30 @@ -118,6 +118,135 @@ def __init__(self, session, sleep=time.sleep): self._sleep = sleep self._client_cache = {} # type: Dict[str, Any] + # establish the endpoint resolver using the botocore loader api + # in order to determine partition and endpoint information + loader = create_loader('data_loader') + endpoints = loader.load_data('endpoints') + self._endpoint_resolver = EndpointResolver(endpoints) + + def resolve_endpoint(self, service, region): + # type: (str, str) -> Union[OrderedDict[str, Any], None] + """Find details of an endpoint based on the service and region. + + This utilizes the botocore EndpointResolver in order to find details on + the given service and region combination. If the service and region + combination is not found the None will be returned. + """ + return self._endpoint_resolver.construct_endpoint(service, region) + + def endpoint_from_arn(self, arn): + # type: (str) -> Union[OrderedDict[str, Any], None] + """Find details for the endpoint associated with a resource ARN. + + This allows the an endpoint to be discerned based on an ARN. This + is a convenience method due to the need to parse multiple ARNs + throughout the project. If the service and region combination + is not found the None will be returned. + """ + arn_split = arn.split(':') + return self.resolve_endpoint(arn_split[2], arn_split[3]) + + def endpoint_dns_suffix(self, service, region): + # type: (str, str) -> str + """Discover the dns suffix for a given service and region combination. + + This allows the service DNS suffix to be discoverable throughout the + framework. If the ARN's service and region combination is not found + then amazonaws.com is returned. + + """ + endpoint = self.resolve_endpoint(service, region) + return endpoint['dnsSuffix'] if endpoint else 'amazonaws.com' + + def endpoint_dns_suffix_from_arn(self, arn): + # type: (str) -> str + """Discover the dns suffix for a given ARN. + + This allows the service DNS suffix to be discoverable throughout the + framework based on the ARN. If the ARN's service and region + combination is not found then amazonaws.com is returned. + + """ + endpoint = self.endpoint_from_arn(arn) + return endpoint['dnsSuffix'] if endpoint else 'amazonaws.com' + + def service_principal(self, service, region='us-east-1', + url_suffix='amazonaws.com'): + # type: (str, str, str) -> str + # Disable too-many-return-statements due to ported code + # pylint: disable=too-many-return-statements + """Compute a "standard" AWS Service principal for given arguments. + + Attribution: This code was ported from https://github.com/aws/aws-cdk + and more specifically, aws-cdk/region-info/lib/default.ts + + Computes a "standard" AWS Service principal for a given service, region + and suffix. This is useful for example when you need to compute a + service principal name, but you do not have a synthesize-time region + literal available (so all you have is `{ "Ref": "AWS::Region" }`). This + way you get the same defaulting behavior that is normally used for + built-in data. + + :param service: the name of the service (s3, s3.amazonaws.com, ...) + :param region: the region in which the service principal is needed. + :param url_suffix: the URL suffix for the partition in which the region + is located. + :return: The service principal for the given combination of arguments + """ + matches = re.match( + ( + r'^([^.]+)' + r'(?:(?:\.amazonaws\.com(?:\.cn)?)|' + r'(?:\.c2s\.ic\.gov)|' + r'(?:\.sc2s\.sgov\.gov))?$' + ), service) + + if matches is None: + # Return "service" if it does not look like any of the following: + # - s3 + # - s3.amazonaws.com + # - s3.amazonaws.com.cn + # - s3.c2s.ic.gov + # - s3.sc2s.sgov.gov + return service + + # Simplify the service name down to something like "s3" + service_name = matches.group(1) + + # Exceptions for Service Principals in us-iso-* + us_iso_exceptions = {'cloudhsm', 'config', 'states', 'workspaces'} + + # Exceptions for Service Principals in us-isob-* + us_isob_exceptions = {'dms', 'states'} + + # Account for idiosyncratic Service Principals in `us-iso-*` regions + if region.startswith('us-iso-') and service_name in us_iso_exceptions: + if service_name == 'states': + # Services with universal principal + return '{}.amazonaws.com'.format(service_name) + else: + # Services with a partitional principal + return '{}.{}'.format(service_name, url_suffix) + + # Account for idiosyncratic Service Principals in `us-isob-*` regions + if region.startswith('us-isob-') and \ + service_name in us_isob_exceptions: + if service_name == 'states': + # Services with universal principal + return '{}.amazonaws.com'.format(service_name) + else: + # Services with a partitional principal + return '{}.{}'.format(service_name, url_suffix) + + if service_name in ['codedeploy', 'logs']: + return '{}.{}.{}'.format(service_name, region, url_suffix) + elif service_name == 'states': + return '{}.{}.amazonaws.com'.format(service_name, region) + elif service_name == 'ec2': + return '{}.{}'.format(service_name, url_suffix) + else: + return '{}.amazonaws.com'.format(service_name) + def lambda_function_exists(self, name): # type: (str) -> bool client = self._client('lambda') @@ -150,18 +279,18 @@ def _create_vpc_config(self, security_group_ids, subnet_ids): return vpc_config def create_function(self, - function_name, # type: str - role_arn, # type: str - zip_contents, # type: str - runtime, # type: str - handler, # type: str + function_name, # type: str + role_arn, # type: str + zip_contents, # type: str + runtime, # type: str + handler, # type: str environment_variables=None, # type: StrMap - tags=None, # type: StrMap - timeout=None, # type: OptInt - memory_size=None, # type: OptInt - security_group_ids=None, # type: OptStrList - subnet_ids=None, # type: OptStrList - layers=None, # type: OptStrList + tags=None, # type: StrMap + timeout=None, # type: OptInt + memory_size=None, # type: OptInt + security_group_ids=None, # type: OptStrList + subnet_ids=None, # type: OptStrList + layers=None, # type: OptStrList ): # type: (...) -> str kwargs = { @@ -270,17 +399,17 @@ def delete_function(self, function_name): raise ResourceDoesNotExistError(function_name) def update_function(self, - function_name, # type: str - zip_contents, # type: str + function_name, # type: str + zip_contents, # type: str environment_variables=None, # type: StrMap - runtime=None, # type: OptStr - tags=None, # type: StrMap - timeout=None, # type: OptInt - memory_size=None, # type: OptInt - role_arn=None, # type: OptStr - subnet_ids=None, # type: OptStrList - security_group_ids=None, # type: OptStrList - layers=None, # type: OptStrList + runtime=None, # type: OptStr + tags=None, # type: StrMap + timeout=None, # type: OptInt + memory_size=None, # type: OptInt + role_arn=None, # type: OptStr + subnet_ids=None, # type: OptStrList + security_group_ids=None, # type: OptStrList + layers=None, # type: OptStrList ): # type: (...) -> Dict[str, Any] """Update a Lambda function's code and configuration. @@ -336,14 +465,14 @@ def delete_function_concurrency(self, function_name): def _update_function_config(self, environment_variables, # type: StrMap - runtime, # type: OptStr - timeout, # type: OptInt - memory_size, # type: OptInt - role_arn, # type: OptStr - subnet_ids, # type: OptStrList - security_group_ids, # type: OptStrList - function_name, # type: str - layers, # type: OptStrList + runtime, # type: OptStr + timeout, # type: OptInt + memory_size, # type: OptInt + role_arn, # type: OptStr + subnet_ids, # type: OptStrList + security_group_ids, # type: OptStrList + function_name, # type: str + layers, # type: OptStrList ): # type: (...) -> None kwargs = {} # type: Dict[str, Any] @@ -675,14 +804,20 @@ def remove_permission_for_sns_topic(self, topic_arn, function_arn): def _build_source_arn_str(self, region_name, account_id, rest_api_id): # type: (str, str, str) -> str source_arn = ( - 'arn:aws:execute-api:' + 'arn:{partition}:execute-api:' '{region_name}:{account_id}:{rest_api_id}/*').format( - region_name=region_name, - # Assuming same account id for lambda function and API gateway. - account_id=account_id, - rest_api_id=rest_api_id) + partition=self.partition_name, + region_name=region_name, + # Assuming same account id for lambda function and API gateway. + account_id=account_id, + rest_api_id=rest_api_id) return source_arn + @property + def partition_name(self): + # type: () -> str + return self._client('apigateway').meta.partition + @property def region_name(self): # type: () -> str @@ -724,7 +859,7 @@ def _convert_to_datetime(self, integer_timestamp): return datetime.utcfromtimestamp(integer_timestamp / 1000.0) def filter_log_events(self, - log_group_name, # type: str + log_group_name, # type: str start_time=None, # type: Optional[datetime] next_token=None, # type: Optional[str] ): @@ -779,18 +914,23 @@ def add_permission_for_authorizer(self, rest_api_id, function_arn, "Unable to find authorizer associated " "with function ARN: %s" % function_arn) parts = function_arn.split(':') + partition = parts[1] region_name = parts[3] account_id = parts[4] function_name = parts[-1] - source_arn = ("arn:aws:execute-api:%s:%s:%s/authorizers/%s" % - (region_name, account_id, rest_api_id, authorizer_id)) + source_arn = ("arn:%s:execute-api:%s:%s:%s/authorizers/%s" % + (partition, region_name, account_id, rest_api_id, + authorizer_id)) + dns_suffix = self.endpoint_dns_suffix('apigateway', region_name) if random_id is None: random_id = self._random_id() self._client('lambda').add_permission( Action='lambda:InvokeFunction', FunctionName=function_name, StatementId=random_id, - Principal='apigateway.amazonaws.com', + Principal=self.service_principal('apigateway', + self.region_name, + dns_suffix), SourceArn=source_arn, ) @@ -904,7 +1044,8 @@ def _merge_s3_notification_config(self, existing_config, new_config): def add_permission_for_s3_event(self, bucket, function_arn): # type: (str, str) -> None - bucket_arn = 'arn:aws:s3:::%s' % bucket + bucket_arn = 'arn:{partition}:s3:::{bucket}'.format( + partition=self.partition_name, bucket=bucket) self._add_lambda_permission_if_needed( source_arn=bucket_arn, function_arn=function_arn, @@ -913,7 +1054,8 @@ def add_permission_for_s3_event(self, bucket, function_arn): def remove_permission_for_s3_event(self, bucket, function_arn): # type: (str, str) -> None - bucket_arn = 'arn:aws:s3:::%s' % bucket + bucket_arn = 'arn:{partition}:s3:::{bucket}'.format( + partition=self.partition_name, bucket=bucket) self._remove_lambda_permission_if_needed( source_arn=bucket_arn, function_arn=function_arn, @@ -946,11 +1088,14 @@ def _add_lambda_permission_if_needed(self, source_arn, function_arn, if self._policy_gives_access(policy, source_arn, service_name): return random_id = self._random_id() + dns_suffix = self.endpoint_dns_suffix_from_arn(source_arn) self._client('lambda').add_permission( Action='lambda:InvokeFunction', FunctionName=function_arn, StatementId=random_id, - Principal='%s.amazonaws.com' % service_name, + Principal=self.service_principal(service_name, + self.region_name, + dns_suffix), SourceArn=source_arn, ) @@ -987,13 +1132,16 @@ def _policy_gives_access(self, policy, source_arn, service_name): def _statement_gives_arn_access(self, statement, source_arn, service_name): # type: (Dict[str, Any], str, str) -> bool + dns_suffix = self.endpoint_dns_suffix_from_arn(source_arn) + principal = self.service_principal(service_name, + self.region_name, + dns_suffix) if not statement['Action'] == 'lambda:InvokeFunction': return False if statement.get('Condition', {}).get( 'ArnLike', {}).get('AWS:SourceArn', '') != source_arn: return False - if statement.get('Principal', {}).get('Service', '') != \ - '%s.amazonaws.com' % service_name: + if statement.get('Principal', {}).get('Service', '') != principal: return False # We're not checking the "Resource" key because we're assuming # that lambda.get_policy() is returning the policy for the particular @@ -1066,9 +1214,9 @@ def verify_event_source_current(self, event_uuid, resource_name, attributes = client.get_event_source_mapping(UUID=event_uuid) actual_arn = attributes['EventSourceArn'] arn_start, actual_name = actual_arn.rsplit(':', 1) - return ( + return bool( actual_name == resource_name and - arn_start.startswith('arn:aws:%s' % service_name) and + re.match("^arn:aws[a-z\\-]*:%s" % service_name, arn_start) and attributes['FunctionArn'] == function_arn ) except client.exceptions.ResourceNotFoundException: @@ -1132,7 +1280,7 @@ def create_websocket_integration( )['IntegrationId'] def create_websocket_route(self, api_id, route_key, integration_id): - # type: (str, str, str, ) -> None + # type: (str, str, str) -> None client = self._client('apigatewayv2') client.create_route( ApiId=api_id, @@ -1170,7 +1318,7 @@ def get_websocket_routes(self, api_id): # type: (str) -> List[str] client = self._client('apigatewayv2') return [i['RouteId'] - for i in client.get_routes(ApiId=api_id,)['Items']] + for i in client.get_routes(ApiId=api_id, )['Items']] def get_websocket_integrations(self, api_id): # type: (str) -> List[str] @@ -1188,12 +1336,12 @@ def create_stage(self, api_id, stage_name, deployment_id): ) def _call_client_method_with_retries( - self, - method, # type: ClientMethod - kwargs, # type: Dict[str, Any] - max_attempts, # type: int - should_retry=None, # type: Callable[[Exception], bool] - delay_time=DELAY_TIME, # type: int + self, + method, # type: ClientMethod + kwargs, # type: Dict[str, Any] + max_attempts, # type: int + should_retry=None, # type: Callable[[Exception], bool] + delay_time=DELAY_TIME, # type: int ): # type: (...) -> Dict[str, Any] client = self._client('lambda') diff --git a/chalice/cli/__init__.py b/chalice/cli/__init__.py index ace4bcdc4f..6b630dabbb 100644 --- a/chalice/cli/__init__.py +++ b/chalice/cli/__init__.py @@ -530,14 +530,19 @@ def generate_models(ctx, stage): type=click.Choice(['json', 'yaml'], case_sensitive=False), help=('Specify if the generated template should be serialized ' 'as either JSON or YAML. CloudFormation only.')) +@click.option('--profile', help='Override profile at packaging time.') @click.argument('out') @click.pass_context def package(ctx, single_file, stage, merge_template, - out, pkg_format, template_format): - # type: (click.Context, bool, str, str, str, str, str) -> None + out, pkg_format, template_format, profile): + # type: (click.Context, bool, str, str, str, str, str, str) -> None factory = ctx.obj['factory'] # type: CLIFactory + factory.profile = profile config = factory.create_config_obj(stage) - packager = factory.create_app_packager(config, pkg_format, template_format, + options = factory.create_package_options() + packager = factory.create_app_packager(config, options, + pkg_format, + template_format, merge_template) if pkg_format == 'terraform' and (merge_template or single_file or diff --git a/chalice/cli/factory.py b/chalice/cli/factory.py index 328f17b50a..a0c0480120 100644 --- a/chalice/cli/factory.py +++ b/chalice/cli/factory.py @@ -1,37 +1,36 @@ -import sys -import os -import json +import functools import importlib +import json import logging -import functools +import os +import sys +from typing import Any, Optional, Dict, MutableMapping, cast # noqa import click from botocore.config import Config as BotocoreConfig from botocore.session import Session -from typing import Any, Optional, Dict, MutableMapping, cast # noqa from chalice import __version__ as chalice_version -from chalice.awsclient import TypedAWSClient +from chalice import local from chalice.app import Chalice # noqa +from chalice.awsclient import TypedAWSClient from chalice.config import Config from chalice.config import DeployedResources # noqa -from chalice.package import create_app_packager -from chalice.package import AppPackager # noqa -from chalice.constants import DEFAULT_STAGE_NAME from chalice.constants import DEFAULT_APIGATEWAY_STAGE_NAME from chalice.constants import DEFAULT_ENDPOINT_TYPE -from chalice.logs import LogRetriever, LogEventGenerator -from chalice.logs import FollowLogEventGenerator -from chalice.logs import BaseLogEventGenerator -from chalice import local -from chalice.utils import UI # noqa -from chalice.utils import PipeReader # noqa +from chalice.constants import DEFAULT_STAGE_NAME from chalice.deploy import deployer # noqa from chalice.deploy import validate from chalice.invoke import LambdaInvokeHandler from chalice.invoke import LambdaInvoker from chalice.invoke import LambdaResponseFormatter - +from chalice.logs import BaseLogEventGenerator +from chalice.logs import FollowLogEventGenerator +from chalice.logs import LogRetriever, LogEventGenerator +from chalice.package import AppPackager # noqa +from chalice.package import create_app_packager, PackageOptions +from chalice.utils import PipeReader # noqa +from chalice.utils import UI # noqa OptStr = Optional[str] OptInt = Optional[int] @@ -75,6 +74,7 @@ def _inject_large_request_body_filter(): class NoSuchFunctionError(Exception): """The specified function could not be found.""" + def __init__(self, name): # type: (str) -> None self.name = name @@ -185,11 +185,11 @@ def _validate_config_from_disk(self, config): except ValueError: raise UnknownConfigFileVersion(string_version) - def create_app_packager(self, config, package_format, template_format, - merge_template=None): - # type: (Config, str, str, OptStr) -> AppPackager + def create_app_packager(self, config, options, package_format, + template_format, merge_template=None): + # type: (Config, PackageOptions, str, str, OptStr) -> AppPackager return create_app_packager( - config, package_format, template_format, + config, options, package_format, template_format, merge_template=merge_template) def create_log_retriever(self, session, lambda_arn, follow_logs): @@ -298,3 +298,10 @@ def load_project_config(self): def create_local_server(self, app_obj, config, host, port): # type: (Chalice, Config, str, int) -> local.LocalDevServer return local.create_local_server(app_obj, config, host, port) + + def create_package_options(self): + # type: () -> PackageOptions + """Create the package options that are required to target regions.""" + s = Session(profile=self.profile) + client = TypedAWSClient(session=s) + return PackageOptions(client) diff --git a/chalice/constants.py b/chalice/constants.py index 6a9530f6e6..edf265e828 100644 --- a/chalice/constants.py +++ b/chalice/constants.py @@ -78,7 +78,7 @@ def index(): "logs:CreateLogStream", "logs:PutLogEvents" ], - "Resource": "arn:aws:logs:*:*:*" + "Resource": "arn:*:logs:*:*:*" } @@ -113,7 +113,7 @@ def index(): "s3:GetObjectVersion", "s3:PutObject" ], - "Resource": "arn:aws:s3:::*", + "Resource": "arn:*:s3:::*", "Effect": "Allow" } ] @@ -247,5 +247,5 @@ def index(): "Action": [ "execute-api:ManageConnections" ], - "Resource": "arn:aws:execute-api:*:*:*/@connections/*" + "Resource": "arn:*:execute-api:*:*:*/@connections/*" } diff --git a/chalice/deploy/appgraph.py b/chalice/deploy/appgraph.py index 7e0c9620ec..95c420012b 100644 --- a/chalice/deploy/appgraph.py +++ b/chalice/deploy/appgraph.py @@ -137,7 +137,7 @@ def _get_default_private_api_policy(self, config): "Effect": "Allow", "Principal": "*", "Action": "execute-api:Invoke", - "Resource": "arn:aws:execute-api:*:*:*", + "Resource": "arn:*:execute-api:*:*:*", "Condition": { "StringEquals": { "aws:SourceVpce": config.api_gateway_endpoint_vpce diff --git a/chalice/deploy/executor.py b/chalice/deploy/executor.py index 5ed47cdf14..6d791848a8 100644 --- a/chalice/deploy/executor.py +++ b/chalice/deploy/executor.py @@ -1,13 +1,13 @@ -import re import pprint +import re +from typing import Dict, List, Any # noqa import jmespath from attr import asdict -from typing import Dict, List, Any # noqa +from chalice.awsclient import TypedAWSClient # noqa from chalice.deploy import models from chalice.deploy.planner import Variable, StringFormat -from chalice.awsclient import TypedAWSClient # noqa from chalice.utils import UI # noqa @@ -114,9 +114,34 @@ def _do_builtinfunction(self, instruction): value = resolved_args[0] parts = value.split(':') result = { + 'partition': parts[1], 'service': parts[2], 'region': parts[3], 'account_id': parts[4], + 'dns_suffix': self._client.endpoint_dns_suffix(parts[2], + parts[3]) + } + self.variables[instruction.output_var] = result + elif instruction.function_name == 'interrogate_profile': + region = self._client.region_name + result = { + 'partition': self._client.partition_name, + 'region': region, + 'dns_suffix': self._client.endpoint_dns_suffix('apigateway', + region) + } + self.variables[instruction.output_var] = result + elif instruction.function_name == 'service_principal': + resolved_args = self._variable_resolver.resolve_variables( + instruction.args, self.variables) + service_name = resolved_args[0] + region_name = self._client.region_name + dns_suffix = self._client.endpoint_dns_suffix(service_name, + region_name) + result = { + 'principal': self._client.service_principal(service_name, + region_name, + dns_suffix) } self.variables[instruction.output_var] = result else: @@ -167,7 +192,6 @@ def resolve_variables(self, value, variables): # The dev commands don't have any backwards compatibility guarantees # so we can alter this output as needed. class DisplayOnlyExecutor(BaseExecutor): - # Max length of bytes object before we truncate with '' _MAX_BYTE_LENGTH = 30 _LINE_VERTICAL = u'\u2502' diff --git a/chalice/deploy/planner.py b/chalice/deploy/planner.py index c531dfacce..fa9d271cfd 100644 --- a/chalice/deploy/planner.py +++ b/chalice/deploy/planner.py @@ -1,15 +1,14 @@ # pylint: disable=too-many-lines import json +import re from collections import OrderedDict - from typing import List, Dict, Any, Optional, Union, Tuple, Set, cast # noqa from typing import Sequence # noqa +from chalice.awsclient import TypedAWSClient, ResourceDoesNotExistError # noqa from chalice.config import Config, DeployedResources # noqa -from chalice.utils import OSUtils # noqa from chalice.deploy import models -from chalice.awsclient import TypedAWSClient, ResourceDoesNotExistError # noqa - +from chalice.utils import OSUtils # noqa InstructionMsg = Union[models.Instruction, Tuple[models.Instruction, str]] MarkedResource = Dict[str, List[models.RecordResource]] @@ -142,8 +141,8 @@ def execute(self, resources): return models.Plan(plan, messages) def _add_result_to_plan(self, - result, # type: Sequence[InstructionMsg] - plan, # type: List[models.Instruction] + result, # type: Sequence[InstructionMsg] + plan, # type: List[models.Instruction] messages, # type: Dict[int, str] ): # type: (...) -> None @@ -266,10 +265,37 @@ def _plan_managediamrole(self, resource): varname = '%s_role_arn' % resource.role_name if not role_exists: return [ + models.BuiltinFunction( + 'service_principal', + ['lambda'], + output_var='lambda_service_principal', + ), + models.JPSearch('principal', + input_var='lambda_service_principal', + output_var='lambda_principal'), + models.StoreValue( + name='lambda_principal', + value=StringFormat('{lambda_principal}', + ['lambda_principal']), + ), + models.StoreValue( + name='lambda_trust_policy', + value={ + "Version": "2012-10-17", + "Statement": [{ + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": Variable('lambda_principal') + }, + "Action": "sts:AssumeRole" + }] + }, + ), (models.APICall( method_name='create_role', params={'name': resource.role_name, - 'trust_policy': resource.trust_policy, + 'trust_policy': Variable('lambda_trust_policy'), 'policy': document}, output_var=varname, ), "Creating IAM role: %s\n" % resource.role_name), @@ -319,7 +345,7 @@ def _plan_snslambdasubscription(self, resource): subscribe_varname = '%s_subscription_arn' % resource.resource_name instruction_for_topic_arn = [] # type: List[InstructionMsg] - if resource.topic.startswith('arn:aws:sns:'): + if re.match(r"^arn:aws[a-z\-]*:sns:", resource.topic): instruction_for_topic_arn += [ models.StoreValue( name=topic_arn_varname, @@ -342,13 +368,16 @@ def _plan_snslambdasubscription(self, resource): models.JPSearch('region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch('partition', + input_var='parsed_lambda_arn', + output_var='partition'), models.StoreValue( name=topic_arn_varname, value=StringFormat( - 'arn:aws:sns:{region_name}:{account_id}:%s' % ( + 'arn:{partition}:sns:{region_name}:{account_id}:%s' % ( resource.topic ), - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), ] @@ -445,13 +474,16 @@ def _plan_sqseventsource(self, resource): models.JPSearch('region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch('partition', + input_var='parsed_lambda_arn', + output_var='partition'), models.StoreValue( name=queue_arn_varname, value=StringFormat( - 'arn:aws:sqs:{region_name}:{account_id}:%s' % ( + 'arn:{partition}:sqs:{region_name}:{account_id}:%s' % ( resource.queue ), - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), ] # type: List[InstructionMsg] @@ -646,11 +678,11 @@ def _inject_websocket_integrations(self, configs): models.StoreValue( name='websocket-%s-integration-lambda-path' % key, value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' - 'invocations' % config['name'], - ['region_name', 'account_id'], + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}' + ':lambda:{region_name}:{account_id}:function' + ':%s/invocations' % config['name'], + ['partition', 'region_name', 'account_id'], ), ), ) @@ -707,6 +739,12 @@ def _plan_websocketapi(self, resource): models.JPSearch('region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch('partition', + input_var='parsed_lambda_arn', + output_var='partition'), + models.JPSearch('dns_suffix', + input_var='parsed_lambda_arn', + output_var='dns_suffix'), ] # type: List[InstructionMsg] # There's also a set of instructions that are needed @@ -717,8 +755,8 @@ def _plan_websocketapi(self, resource): name='websocket_api_url', value=StringFormat( 'wss://{websocket_api_id}.execute-api.{region_name}' - '.amazonaws.com/%s/' % resource.api_gateway_stage, - ['websocket_api_id', 'region_name'], + '.{dns_suffix}/%s/' % resource.api_gateway_stage, + ['websocket_api_id', 'region_name', 'dns_suffix'], ), ), models.RecordResourceVariable( @@ -850,6 +888,12 @@ def _plan_restapi(self, resource): models.JPSearch('region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch('partition', + input_var='parsed_lambda_arn', + output_var='partition'), + models.JPSearch('dns_suffix', + input_var='parsed_lambda_arn', + output_var='dns_suffix'), # The swagger doc uses the 'api_handler_lambda_arn' # var name so we need to make sure we populate this variable # before importing the rest API. @@ -889,8 +933,8 @@ def _plan_restapi(self, resource): name='rest_api_url', value=StringFormat( 'https://{rest_api_id}.execute-api.{region_name}' - '.amazonaws.com/%s/' % resource.api_gateway_stage, - ['rest_api_id', 'region_name'], + '.{dns_suffix}/%s/' % resource.api_gateway_stage, + ['rest_api_id', 'region_name', 'dns_suffix'], ), ), models.RecordResourceVariable( diff --git a/chalice/deploy/swagger.py b/chalice/deploy/swagger.py index d89d459231..c12a994ba3 100644 --- a/chalice/deploy/swagger.py +++ b/chalice/deploy/swagger.py @@ -172,9 +172,11 @@ def _uri(self, lambda_arn=None): # type: (Optional[str]) -> Any if lambda_arn is None: lambda_arn = self._deployed_resources['api_handler_arn'] - return ('arn:aws:apigateway:{region}:lambda:path/2015-03-31' + partition = lambda_arn.split(':')[1] + return ('arn:{partition}:apigateway:{region}:lambda:path/2015-03-31' '/functions/{lambda_arn}/invocations').format( - region=self._region, lambda_arn=lambda_arn) + partition=partition, region=self._region, + lambda_arn=lambda_arn) def _generate_apig_integ(self, view): # type: (RouteEntry) -> Dict[str, Any] @@ -250,7 +252,8 @@ def _uri(self, lambda_arn=None): # type: (Optional[str]) -> Any return { 'Fn::Sub': ( - 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31' + 'arn:${AWS::Partition}:apigateway:${AWS::Region}' + ':lambda:path/2015-03-31' '/functions/${APIHandler.Arn}/invocations' ) } @@ -259,7 +262,8 @@ def _auth_uri(self, authorizer): # type: (ChaliceAuthorizer) -> Any return { 'Fn::Sub': ( - 'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31' + 'arn:${AWS::Partition}:apigateway:${AWS::Region}' + ':lambda:path/2015-03-31' '/functions/${%s.Arn}/invocations' % to_cfn_resource_name( authorizer.name) ) @@ -274,18 +278,18 @@ def __init__(self): def _uri(self, lambda_arn=None): # type: (Optional[str]) -> Any return StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/2015-03-31' + 'arn:{partition}:apigateway:{region_name}:lambda:path/2015-03-31' '/functions/{api_handler_lambda_arn}/invocations', - ['region_name', 'api_handler_lambda_arn'], + ['partition', 'region_name', 'api_handler_lambda_arn'], ) def _auth_uri(self, authorizer): # type: (ChaliceAuthorizer) -> Any varname = '%s_lambda_arn' % authorizer.name return StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/2015-03-31' + 'arn:{partition}:apigateway:{region_name}:lambda:path/2015-03-31' '/functions/{%s}/invocations' % varname, - ['region_name', varname], + ['partition', 'region_name', varname], ) diff --git a/chalice/package.py b/chalice/package.py index 2fb6e29144..4b54970abe 100644 --- a/chalice/package.py +++ b/chalice/package.py @@ -3,10 +3,12 @@ import copy import json import os +import re import six from typing import Any, Optional, Dict, List, Set, Union # noqa from typing import cast + import yaml from yaml.scanner import ScannerError from yaml.nodes import Node, ScalarNode, SequenceNode @@ -16,6 +18,7 @@ from chalice.utils import ( OSUtils, UI, serialize_to_json, to_cfn_resource_name ) +from chalice.awsclient import TypedAWSClient # noqa from chalice.config import Config # noqa from chalice.deploy import models from chalice.deploy.appgraph import ApplicationGraphBuilder, DependencyBuilder @@ -24,9 +27,9 @@ def create_app_packager( - config, package_format='cloudformation', + config, options, package_format='cloudformation', template_format='json', merge_template=None): - # type: (Config, str, str, Optional[str]) -> AppPackager + # type: (Config, PackageOptions, str, str, Optional[str]) -> AppPackager osutils = OSUtils() ui = UI() application_builder = ApplicationGraphBuilder() @@ -54,11 +57,11 @@ def create_app_packager( merger=TemplateDeepMerger(), template_serializer=template_serializer, merge_template=merge_template)]) - generator = SAMTemplateGenerator(config) + generator = SAMTemplateGenerator(config, options) else: build_stage = create_build_stage( osutils, ui, TerraformSwaggerGenerator()) - generator = TerraformGenerator(config) + generator = TerraformGenerator(config, options) post_processors.append( TerraformCodeLocationPostProcessor(osutils=osutils)) @@ -81,6 +84,20 @@ class DuplicateResourceNameError(Exception): pass +class PackageOptions(object): + def __init__(self, client): + # type: (TypedAWSClient) -> None + self._client = client # type: TypedAWSClient + + def service_principal(self, service): + # type: (str) -> str + dns_suffix = self._client.endpoint_dns_suffix(service, + self._client.region_name) + return self._client.service_principal(service, + self._client.region_name, + dns_suffix) + + class ResourceBuilder(object): def __init__(self, application_builder, # type: ApplicationGraphBuilder @@ -105,9 +122,10 @@ class TemplateGenerator(object): template_file = None # type: str - def __init__(self, config): - # type: (Config) -> None + def __init__(self, config, options): + # type: (Config, PackageOptions) -> None self._config = config + self._options = options def dispatch(self, resource, template): # type: (models.Model, Dict[str, Any]) -> None @@ -141,7 +159,6 @@ def _default(self, resource, template): class SAMTemplateGenerator(TemplateGenerator): - _BASE_TEMPLATE = { 'AWSTemplateFormatVersion': '2010-09-09', 'Transform': 'AWS::Serverless-2016-10-31', @@ -151,9 +168,9 @@ class SAMTemplateGenerator(TemplateGenerator): template_file = "sam" - def __init__(self, config): - # type: (Config) -> None - super(SAMTemplateGenerator, self).__init__(config) + def __init__(self, config, options): + # type: (Config, PackageOptions) -> None + super(SAMTemplateGenerator, self).__init__(config, options) self._seen_names = set([]) # type: Set[str] def generate(self, resources): @@ -284,11 +301,11 @@ def _generate_restapi(self, resource, template): 'Properties': { 'FunctionName': {'Ref': 'APIHandler'}, 'Action': 'lambda:InvokeFunction', - 'Principal': 'apigateway.amazonaws.com', + 'Principal': self._options.service_principal('apigateway'), 'SourceArn': { 'Fn::Sub': [ - ('arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}' - ':${RestAPIId}/*'), + ('arn:${AWS::Partition}:execute-api:${AWS::Region}' + ':${AWS::AccountId}:${RestAPIId}/*'), {'RestAPIId': {'Ref': 'RestAPI'}}, ] }, @@ -301,11 +318,12 @@ def _generate_restapi(self, resource, template): 'Properties': { 'FunctionName': {'Fn::GetAtt': [auth_cfn_name, 'Arn']}, 'Action': 'lambda:InvokeFunction', - 'Principal': 'apigateway.amazonaws.com', + 'Principal': self._options.service_principal('apigateway'), 'SourceArn': { 'Fn::Sub': [ - ('arn:aws:execute-api:${AWS::Region}:' - '${AWS::AccountId}:${RestAPIId}/*'), + ('arn:${AWS::Partition}:execute-api' + ':${AWS::Region}:${AWS::AccountId}' + ':${RestAPIId}/*'), {'RestAPIId': {'Ref': 'RestAPI'}}, ] }, @@ -342,7 +360,7 @@ def _inject_restapi_outputs(self, template): 'https://${RestAPI}.execute-api.${AWS::Region}' # The api_gateway_stage is filled in when # the template is built. - '.amazonaws.com/%s/' + '.${AWS::URLSuffix}/%s/' ) % stage_name } } @@ -360,10 +378,11 @@ def _add_websocket_lambda_integration( 'IntegrationUri': { 'Fn::Sub': [ ( - 'arn:aws:apigateway:${AWS::Region}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:' - '${AWS::Region}:' '${AWS::AccountId}:function:' - '${WebsocketHandler}/invocations' + 'arn:${AWS::Partition}:apigateway:${AWS::Region}' + ':lambda:path/2015-03-31/functions/arn' + ':${AWS::Partition}:lambda:${AWS::Region}' + ':${AWS::AccountId}:function' + ':${WebsocketHandler}/invocations' ), {'WebsocketHandler': {'Ref': websocket_handler}} ], @@ -379,10 +398,11 @@ def _add_websocket_lambda_invoke_permission( 'Properties': { 'FunctionName': {'Ref': websocket_handler}, 'Action': 'lambda:InvokeFunction', - 'Principal': 'apigateway.amazonaws.com', + 'Principal': self._options.service_principal('apigateway'), 'SourceArn': { 'Fn::Sub': [ - ('arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}' + ('arn:${AWS::Partition}:execute-api' + ':${AWS::Region}:${AWS::AccountId}' ':${WebsocketAPIId}/*'), {'WebsocketAPIId': api_ref}, ], @@ -514,7 +534,7 @@ def _inject_websocketapi_outputs(self, template): 'wss://${WebsocketAPI}.execute-api.${AWS::Region}' # The api_gateway_stage is filled in when # the template is built. - '.amazonaws.com/%s/' + '.${AWS::URLSuffix}/%s/' ) % stage_name } } @@ -528,6 +548,8 @@ def _generate_managediamrole(self, resource, template): # type: (models.ManagedIAMRole, Dict[str, Any]) -> None role_cfn_name = self._register_cfn_resource_name( resource.resource_name) + resource.trust_policy['Statement'][0]['Principal']['Service'] = \ + self._options.service_principal('lambda') template['Resources'][role_cfn_name] = { 'Type': 'AWS::IAM::Role', 'Properties': { @@ -557,12 +579,13 @@ def _generate_snslambdasubscription(self, resource, template): sns_cfn_name = self._register_cfn_resource_name( resource.resource_name) - if resource.topic.startswith('arn:aws:sns:'): + if re.match(r"^arn:aws[a-z\-]*:sns:", resource.topic): topic_arn = resource.topic # type: Union[str, Dict[str, str]] else: topic_arn = { 'Fn::Sub': ( - 'arn:aws:sns:${AWS::Region}:${AWS::AccountId}:%s' % + 'arn:${AWS::Partition}:sns' + ':${AWS::Region}:${AWS::AccountId}:%s' % resource.topic ) } @@ -588,7 +611,8 @@ def _generate_sqseventsource(self, resource, template): 'Properties': { 'Queue': { 'Fn::Sub': ( - 'arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:%s' % + 'arn:${AWS::Partition}:sqs:${AWS::Region}' + ':${AWS::AccountId}:%s' % resource.queue ) }, @@ -610,7 +634,6 @@ def _register_cfn_resource_name(self, name): class TerraformGenerator(TemplateGenerator): - template_file = "chalice.tf" def generate(self, resources): @@ -627,6 +650,7 @@ def generate(self, resources): }, 'data': { 'aws_caller_identity': {'chalice': {}}, + 'aws_partition': {'chalice': {}}, 'aws_region': {'chalice': {}}, 'null_data_source': { 'chalice': { @@ -651,6 +675,7 @@ def _fref(self, lambda_function, attr='arn'): def _arnref(self, arn_template, **kw): # type: (str, str) -> str d = dict( + partition='${data.aws_partition.chalice.partition}', region='${data.aws_region.chalice.name}', account_id='${data.aws_caller_identity.chalice.account_id}') d.update(kw) @@ -658,17 +683,21 @@ def _arnref(self, arn_template, **kw): def _generate_managediamrole(self, resource, template): # type: (models.ManagedIAMRole, Dict[str, Any]) -> None + + resource.trust_policy['Statement'][0]['Principal']['Service'] = \ + self._options.service_principal('lambda') + template['resource'].setdefault('aws_iam_role', {})[ resource.resource_name] = { - 'name': resource.role_name, - 'assume_role_policy': json.dumps(resource.trust_policy) + 'name': resource.role_name, + 'assume_role_policy': json.dumps(resource.trust_policy) } template['resource'].setdefault('aws_iam_role_policy', {})[ resource.resource_name] = { - 'name': resource.resource_name + 'Policy', - 'policy': json.dumps(resource.policy.document), - 'role': '${aws_iam_role.%s.id}' % resource.resource_name, + 'name': resource.resource_name + 'Policy', + 'policy': json.dumps(resource.policy.document), + 'role': '${aws_iam_role.%s.id}' % resource.resource_name, } def _generate_websocketapi(self, resource, template): @@ -706,28 +735,29 @@ def _generate_s3bucketnotification(self, resource, template): bucket_name = resource.bucket template['resource'].setdefault( 'aws_s3_bucket_notification', {}).setdefault( - bucket_name + '_notify', - {'bucket': resource.bucket}).setdefault( - 'lambda_function', []).append(bnotify) + bucket_name + '_notify', + {'bucket': resource.bucket}).setdefault( + 'lambda_function', []).append(bnotify) template['resource'].setdefault('aws_lambda_permission', {})[ resource.resource_name] = { - 'statement_id': resource.resource_name, - 'action': 'lambda:InvokeFunction', - 'function_name': resource.lambda_function.function_name, - 'principal': 's3.amazonaws.com', - 'source_arn': 'arn:aws:s3:::%s' % resource.bucket + 'statement_id': resource.resource_name, + 'action': 'lambda:InvokeFunction', + 'function_name': resource.lambda_function.function_name, + 'principal': self._options.service_principal('s3'), + 'source_arn': 'arn:*:s3:::%s' % resource.bucket } def _generate_sqseventsource(self, resource, template): # type: (models.SQSEventSource, Dict[str, Any]) -> None template['resource'].setdefault('aws_lambda_event_source_mapping', {})[ resource.resource_name] = { - 'event_source_arn': self._arnref( - "arn:aws:sqs:%(region)s:%(account_id)s:%(queue)s", - queue=resource.queue), - 'batch_size': resource.batch_size, - 'function_name': resource.lambda_function.function_name, + 'event_source_arn': self._arnref( + "arn:%(partition)s:sqs:%(region)s" + ":%(account_id)s:%(queue)s", + queue=resource.queue), + 'batch_size': resource.batch_size, + 'function_name': resource.lambda_function.function_name, } def _generate_snslambdasubscription(self, resource, template): @@ -737,21 +767,21 @@ def _generate_snslambdasubscription(self, resource, template): topic_arn = resource.topic else: topic_arn = self._arnref( - 'arn:aws:sns:%(region)s:%(account_id)s:%(topic)s', + 'arn:%(partition)s:sns:%(region)s:%(account_id)s:%(topic)s', topic=resource.topic) template['resource'].setdefault('aws_sns_topic_subscription', {})[ resource.resource_name] = { - 'topic_arn': topic_arn, - 'protocol': 'lambda', - 'endpoint': self._fref(resource.lambda_function) + 'topic_arn': topic_arn, + 'protocol': 'lambda', + 'endpoint': self._fref(resource.lambda_function) } template['resource'].setdefault('aws_lambda_permission', {})[ resource.resource_name] = { - 'function_name': resource.lambda_function.function_name, - 'action': 'lambda:InvokeFunction', - 'principal': 'sns.amazonaws.com', - 'source_arn': topic_arn + 'function_name': resource.lambda_function.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': self._options.service_principal('sns'), + 'source_arn': topic_arn } def _generate_cloudwatchevent(self, resource, template): @@ -759,9 +789,9 @@ def _generate_cloudwatchevent(self, resource, template): template['resource'].setdefault( 'aws_cloudwatch_event_rule', {})[ - resource.resource_name] = { - 'name': resource.resource_name, - 'event_pattern': resource.event_pattern + resource.resource_name] = { + 'name': resource.resource_name, + 'event_pattern': resource.event_pattern } self._cwe_helper(resource, template) @@ -770,10 +800,10 @@ def _generate_scheduledevent(self, resource, template): template['resource'].setdefault( 'aws_cloudwatch_event_rule', {})[ - resource.resource_name] = { - 'name': resource.resource_name, - 'schedule_expression': resource.schedule_expression, - 'description': resource.rule_description, + resource.resource_name] = { + 'name': resource.resource_name, + 'schedule_expression': resource.schedule_expression, + 'description': resource.rule_description, } self._cwe_helper(resource, template) @@ -781,20 +811,20 @@ def _cwe_helper(self, resource, template): # type: (models.CloudWatchEventBase, Dict[str, Any]) -> None template['resource'].setdefault( 'aws_cloudwatch_event_target', {})[ - resource.resource_name] = { - 'rule': '${aws_cloudwatch_event_rule.%s.name}' % ( - resource.resource_name), - 'target_id': resource.resource_name, - 'arn': self._fref(resource.lambda_function) + resource.resource_name] = { + 'rule': '${aws_cloudwatch_event_rule.%s.name}' % ( + resource.resource_name), + 'target_id': resource.resource_name, + 'arn': self._fref(resource.lambda_function) } template['resource'].setdefault( 'aws_lambda_permission', {})[ - resource.resource_name] = { - 'function_name': resource.lambda_function.function_name, - 'action': 'lambda:InvokeFunction', - 'principal': 'events.amazonaws.com', - 'source_arn': "${aws_cloudwatch_event_rule.%s.arn}" % ( - resource.resource_name) + resource.resource_name] = { + 'function_name': resource.lambda_function.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': self._options.service_principal('events'), + 'source_arn': "${aws_cloudwatch_event_rule.%s.arn}" % ( + resource.resource_name) } def _generate_lambdafunction(self, resource, template): @@ -845,82 +875,83 @@ def _generate_restapi(self, resource, template): swagger_doc = cast(Dict, resource.swagger_doc) template['data'].setdefault( 'template_file', {}).setdefault( - 'chalice_api_swagger', {})['template'] = json.dumps( - swagger_doc) + 'chalice_api_swagger', {})['template'] = json.dumps( + swagger_doc) template['resource'].setdefault('aws_api_gateway_rest_api', {})[ resource.resource_name] = { - 'body': '${data.template_file.chalice_api_swagger.rendered}', - # Terraform will diff explicitly configured attributes - # to the current state of the resource. Attributes configured - # via swagger on the REST api need to be duplicated here, else - # terraform will set them back to empty. - 'name': swagger_doc['info']['title'], - 'binary_media_types': swagger_doc[ - 'x-amazon-apigateway-binary-media-types'], - 'endpoint_configuration': {'types': [resource.endpoint_type]} + 'body': '${data.template_file.chalice_api_swagger.rendered}', + # Terraform will diff explicitly configured attributes + # to the current state of the resource. Attributes configured + # via swagger on the REST api need to be duplicated here, else + # terraform will set them back to empty. + 'name': swagger_doc['info']['title'], + 'binary_media_types': swagger_doc[ + 'x-amazon-apigateway-binary-media-types'], + 'endpoint_configuration': {'types': [resource.endpoint_type]} } if 'x-amazon-apigateway-policy' in swagger_doc: template['resource'][ 'aws_api_gateway_rest_api'][ - resource.resource_name]['policy'] = json.dumps( - swagger_doc['x-amazon-apigateway-policy']) + resource.resource_name]['policy'] = json.dumps( + swagger_doc['x-amazon-apigateway-policy']) if resource.minimum_compression.isdigit(): template['resource'][ 'aws_api_gateway_rest_api'][ - resource.resource_name][ - 'minimum_compression_size'] = int( - resource.minimum_compression) + resource.resource_name][ + 'minimum_compression_size'] = int( + resource.minimum_compression) template['resource'].setdefault('aws_api_gateway_deployment', {})[ resource.resource_name] = { - 'stage_name': resource.api_gateway_stage, - # Ensure that the deployment gets redeployed if we update - # the swagger description for the api by using its checksum - # in the stage description. - 'stage_description': ( - "${md5(data.template_file.chalice_api_swagger.rendered)}"), - 'rest_api_id': '${aws_api_gateway_rest_api.%s.id}' % ( - resource.resource_name), - 'lifecycle': {'create_before_destroy': True} + 'stage_name': resource.api_gateway_stage, + # Ensure that the deployment gets redeployed if we update + # the swagger description for the api by using its checksum + # in the stage description. + 'stage_description': ( + "${md5(data.template_file.chalice_api_swagger.rendered)}"), + 'rest_api_id': '${aws_api_gateway_rest_api.%s.id}' % ( + resource.resource_name), + 'lifecycle': {'create_before_destroy': True} } template['resource'].setdefault('aws_lambda_permission', {})[ resource.resource_name + '_invoke'] = { - 'function_name': resource.lambda_function.function_name, - 'action': 'lambda:InvokeFunction', - 'principal': 'apigateway.amazonaws.com', - 'source_arn': - "${aws_api_gateway_rest_api.%s.execution_arn}/*" % ( - resource.resource_name) + 'function_name': resource.lambda_function.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': self._options.service_principal('apigateway'), + 'source_arn': + "${aws_api_gateway_rest_api.%s.execution_arn}/*" % ( + resource.resource_name) } template.setdefault('output', {})[ 'EndpointURL'] = { - 'value': '${aws_api_gateway_deployment.%s.invoke_url}' % ( - resource.resource_name) + 'value': '${aws_api_gateway_deployment.%s.invoke_url}' % ( + resource.resource_name) } for auth in resource.authorizers: template['resource']['aws_lambda_permission'][ auth.resource_name + '_invoke'] = { - 'function_name': auth.function_name, - 'action': 'lambda:InvokeFunction', - 'principal': 'apigateway.amazonaws.com', - 'source_arn': ( - "${aws_api_gateway_rest_api.%s.execution_arn}" % ( - resource.resource_name) + "/*") + 'function_name': auth.function_name, + 'action': 'lambda:InvokeFunction', + 'principal': self._options.service_principal('apigateway'), + 'source_arn': ( + "${aws_api_gateway_rest_api.%s.execution_arn}" % ( + resource.resource_name) + "/*" + ) } class AppPackager(object): def __init__(self, - templater, # type: TemplateGenerator - resource_builder, # type: ResourceBuilder - post_processor, # type: TemplatePostProcessor + templater, # type: TemplateGenerator + resource_builder, # type: ResourceBuilder + post_processor, # type: TemplatePostProcessor template_serializer, # type: TemplateSerializer - osutils, # type: OSUtils + osutils, # type: OSUtils ): # type: (...) -> None self._templater = templater @@ -1013,8 +1044,8 @@ def process(self, template, config, outdir, chalice_stage_name): class TemplateMergePostProcessor(TemplatePostProcessor): def __init__(self, - osutils, # type: OSUtils - merger, # type: TemplateMerger + osutils, # type: OSUtils + merger, # type: TemplateMerger template_serializer, # type: TemplateSerializer merge_template=None, # type: Optional[str] ): @@ -1070,7 +1101,7 @@ def merge(self, file_template, chalice_template): def _merge(self, file_template, chalice_template): # type: (Any, Any) -> Any if isinstance(file_template, dict) and \ - isinstance(chalice_template, dict): + isinstance(chalice_template, dict): return self._merge_dict(file_template, chalice_template) return file_template @@ -1083,7 +1114,6 @@ def _merge_dict(self, file_template, chalice_template): class TemplateSerializer(object): - file_extension = '' def load_template(self, file_contents, filename=''): @@ -1096,7 +1126,6 @@ def serialize_template(self, contents): class JSONTemplateSerializer(TemplateSerializer): - file_extension = 'json' def serialize_template(self, contents): @@ -1113,7 +1142,6 @@ def load_template(self, file_contents, filename=''): class YAMLTemplateSerializer(TemplateSerializer): - file_extension = 'yaml' @classmethod diff --git a/chalice/pipeline.py b/chalice/pipeline.py index a0517ae739..9d3fcff3bc 100644 --- a/chalice/pipeline.py +++ b/chalice/pipeline.py @@ -234,7 +234,7 @@ def _add_codebuild_role(self, resources, outputs): "Effect": "Allow", "Principal": { "Service": [ - "codebuild.amazonaws.com" + {'Fn::Sub': 'codebuild.${AWS::URLSuffix}'} ] } } @@ -307,7 +307,8 @@ def _add_cfn_deploy_role(self, resources, outputs): "Effect": "Allow", "Principal": { "Service": [ - "cloudformation.amazonaws.com" + {'Fn::Sub': + 'cloudformation.${AWS::URLSuffix}'} ] } } @@ -541,7 +542,8 @@ def _add_codepipeline_role(self, resources, outputs): "Effect": "Allow", "Principal": { "Service": [ - "codepipeline.amazonaws.com" + {'Fn::Sub': 'codepipeline' + '.${AWS::URLSuffix}'} ] } } diff --git a/chalice/utils.py b/chalice/utils.py index 42a193a2cf..64a1ffea42 100644 --- a/chalice/utils.py +++ b/chalice/utils.py @@ -11,6 +11,8 @@ from datetime import datetime, timedelta import subprocess + +from collections import OrderedDict # noqa import click from typing import IO, Dict, List, Any, Tuple, Iterator, BinaryIO, Text # noqa from typing import Optional, Union # noqa @@ -21,7 +23,6 @@ from chalice.constants import WELCOME_PROMPT - OptInt = Optional[int] OptStr = Optional[str] EnvVars = MutableMapping diff --git a/chalice/vendored/__init__.py b/chalice/vendored/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/chalice/vendored/botocore/__init__.py b/chalice/vendored/botocore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/chalice/vendored/botocore/regions.py b/chalice/vendored/botocore/regions.py new file mode 100644 index 0000000000..802f90b03d --- /dev/null +++ b/chalice/vendored/botocore/regions.py @@ -0,0 +1,201 @@ +# Copyright 2014 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +"""Resolves regions and endpoints. + +This module implements endpoint resolution, including resolving endpoints for a +given service and region and resolving the available endpoints for a service +in a specific AWS partition. +""" +import logging +import re + +from botocore.exceptions import NoRegionError + +LOG = logging.getLogger(__name__) +DEFAULT_URI_TEMPLATE = '{service}.{region}.{dnsSuffix}' +DEFAULT_SERVICE_DATA = {'endpoints': {}} + + +class BaseEndpointResolver(object): + """Resolves regions and endpoints. Must be subclassed.""" + def construct_endpoint(self, service_name, region_name=None): + """Resolves an endpoint for a service and region combination. + + :type service_name: string + :param service_name: Name of the service to resolve an endpoint for + (e.g., s3) + + :type region_name: string + :param region_name: Region/endpoint name to resolve (e.g., us-east-1) + if no region is provided, the first found partition-wide endpoint + will be used if available. + + :rtype: dict + :return: Returns a dict containing the following keys: + - partition: (string, required) Resolved partition name + - endpointName: (string, required) Resolved endpoint name + - hostname: (string, required) Hostname to use for this endpoint + - sslCommonName: (string) sslCommonName to use for this endpoint. + - credentialScope: (dict) Signature version 4 credential scope + - region: (string) region name override when signing. + - service: (string) service name override when signing. + - signatureVersions: (list) A list of possible signature + versions, including s3, v4, v2, and s3v4 + - protocols: (list) A list of supported protocols + (e.g., http, https) + - ...: Other keys may be included as well based on the metadata + """ + raise NotImplementedError + + def get_available_partitions(self): + """Lists the partitions available to the endpoint resolver. + + :return: Returns a list of partition names (e.g., ["aws", "aws-cn"]). + """ + raise NotImplementedError + + def get_available_endpoints(self, service_name, partition_name='aws', + allow_non_regional=False): + """Lists the endpoint names of a particular partition. + + :type service_name: string + :param service_name: Name of a service to list endpoint for (e.g., s3) + + :type partition_name: string + :param partition_name: Name of the partition to limit endpoints to. + (e.g., aws for the public AWS endpoints, aws-cn for AWS China + endpoints, aws-us-gov for AWS GovCloud (US) Endpoints, etc. + + :type allow_non_regional: bool + :param allow_non_regional: Set to True to include endpoints that are + not regional endpoints (e.g., s3-external-1, + fips-us-gov-west-1, etc). + :return: Returns a list of endpoint names (e.g., ["us-east-1"]). + """ + raise NotImplementedError + + +class EndpointResolver(BaseEndpointResolver): + """Resolves endpoints based on partition endpoint metadata""" + def __init__(self, endpoint_data): + """ + :param endpoint_data: A dict of partition data. + """ + if 'partitions' not in endpoint_data: + raise ValueError('Missing "partitions" in endpoint data') + self._endpoint_data = endpoint_data + + def get_available_partitions(self): + result = [] + for partition in self._endpoint_data['partitions']: + result.append(partition['partition']) + return result + + def get_available_endpoints(self, service_name, partition_name='aws', + allow_non_regional=False): + result = [] + for partition in self._endpoint_data['partitions']: + if partition['partition'] != partition_name: + continue + services = partition['services'] + if service_name not in services: + continue + for endpoint_name in services[service_name]['endpoints']: + if allow_non_regional or endpoint_name in partition['regions']: + result.append(endpoint_name) + return result + + def construct_endpoint(self, service_name, region_name=None, partition_name=None): + if partition_name is not None: + valid_partition = None + for partition in self._endpoint_data['partitions']: + if partition['partition'] == partition_name: + valid_partition = partition + + if valid_partition is not None: + result = self._endpoint_for_partition(valid_partition, service_name, + region_name, True) + return result + return None + + # Iterate over each partition until a match is found. + for partition in self._endpoint_data['partitions']: + result = self._endpoint_for_partition( + partition, service_name, region_name) + if result: + return result + + def _endpoint_for_partition(self, partition, service_name, region_name, + force_partition=False): + # Get the service from the partition, or an empty template. + service_data = partition['services'].get( + service_name, DEFAULT_SERVICE_DATA) + # Use the partition endpoint if no region is supplied. + if region_name is None: + if 'partitionEndpoint' in service_data: + region_name = service_data['partitionEndpoint'] + else: + raise NoRegionError() + # Attempt to resolve the exact region for this partition. + if region_name in service_data['endpoints']: + return self._resolve( + partition, service_name, service_data, region_name) + # Check to see if the endpoint provided is valid for the partition. + if self._region_match(partition, region_name) or force_partition: + # Use the partition endpoint if set and not regionalized. + partition_endpoint = service_data.get('partitionEndpoint') + is_regionalized = service_data.get('isRegionalized', True) + if partition_endpoint and not is_regionalized: + LOG.debug('Using partition endpoint for %s, %s: %s', + service_name, region_name, partition_endpoint) + return self._resolve( + partition, service_name, service_data, partition_endpoint) + LOG.debug('Creating a regex based endpoint for %s, %s', + service_name, region_name) + return self._resolve( + partition, service_name, service_data, region_name) + + def _region_match(self, partition, region_name): + if region_name in partition['regions']: + return True + if 'regionRegex' in partition: + return re.compile(partition['regionRegex']).match(region_name) + return False + + def _resolve(self, partition, service_name, service_data, endpoint_name): + result = service_data['endpoints'].get(endpoint_name, {}) + result['partition'] = partition['partition'] + result['endpointName'] = endpoint_name + # Merge in the service defaults then the partition defaults. + self._merge_keys(service_data.get('defaults', {}), result) + self._merge_keys(partition.get('defaults', {}), result) + hostname = result.get('hostname', DEFAULT_URI_TEMPLATE) + result['hostname'] = self._expand_template( + partition, result['hostname'], service_name, endpoint_name) + if 'sslCommonName' in result: + result['sslCommonName'] = self._expand_template( + partition, result['sslCommonName'], service_name, + endpoint_name) + result['dnsSuffix'] = partition['dnsSuffix'] + return result + + def _merge_keys(self, from_data, result): + for key in from_data: + if key not in result: + result[key] = from_data[key] + + def _expand_template(self, partition, template, service_name, + endpoint_name): + return template.format( + service=service_name, region=endpoint_name, + dnsSuffix=partition['dnsSuffix']) \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 5e4090017a..e77fab171b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,5 @@ [wheel] universal = 1 + +[mypy-chalice.vendored.*] +ignore_errors = true \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 90a052523e..eff1f22214 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -134,7 +134,7 @@ def add_response_error(self, method, error, expected_params=None): """Adds a custom exception to the response queue :type method: str - :param method: Thhe name of the service method to raise the error + :param method: The name of the service method to raise the error on. :type error: Exception diff --git a/tests/functional/cli/test_cli.py b/tests/functional/cli/test_cli.py index 8c10286750..1cba6fed51 100644 --- a/tests/functional/cli/test_cli.py +++ b/tests/functional/cli/test_cli.py @@ -189,8 +189,8 @@ def test_can_write_swagger_model(runner): } }, "uri": ( - "arn:aws:apigateway:{region_name}:lambda:" - "path/2015-03-31/functions/" + "arn:{partition}:apigateway:{region_name}" + ":lambda:path/2015-03-31/functions/" "{api_handler_lambda_arn}/invocations" ), "passthroughBehavior": "when_no_match", @@ -228,7 +228,7 @@ def test_can_write_swagger_model(runner): } -def test_can_package_command(runner): +def test_can_package_command(runner, mock_cli_factory): with runner.isolated_filesystem(): cli.create_new_project_skeleton('testproject') os.chdir('testproject') diff --git a/tests/functional/cli/test_factory.py b/tests/functional/cli/test_factory.py index c8167935a1..d1a2f90e03 100644 --- a/tests/functional/cli/test_factory.py +++ b/tests/functional/cli/test_factory.py @@ -11,6 +11,7 @@ from chalice.config import Config from chalice.config import DeployedResources from chalice import local +from chalice.package import PackageOptions from chalice.utils import UI from chalice import Chalice from chalice.logs import LogRetriever @@ -299,3 +300,8 @@ def test_does_raise_not_found_error_when_resource_is_not_lambda(clifactory): with pytest.raises(factory.NoSuchFunctionError) as e: clifactory.create_lambda_invoke_handler('foobar', stage) assert e.value.name == 'foobar' + + +def test_can_create_package_options(clifactory): + options = clifactory.create_package_options() + assert isinstance(options, PackageOptions) diff --git a/tests/functional/test_awsclient.py b/tests/functional/test_awsclient.py index aa2286861a..2ef4e8a519 100644 --- a/tests/functional/test_awsclient.py +++ b/tests/functional/test_awsclient.py @@ -1893,14 +1893,15 @@ def test_add_permission_for_scheduled_event(stubbed_session): FunctionName='function-arn', StatementId=stub.ANY, Principal='events.amazonaws.com', - SourceArn='rule-arn' + SourceArn='arn:aws:events:us-east-1:123456789012:rule/MyScheduledRule' ).returns({}) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) awsclient.add_permission_for_cloudwatch_event( - 'rule-arn', 'function-arn') + 'arn:aws:events:us-east-1:123456789012:rule/MyScheduledRule', + 'function-arn') stubbed_session.verify_stubs() @@ -1913,7 +1914,8 @@ def test_skip_if_permission_already_granted(stubbed_session): {'Action': 'lambda:InvokeFunction', 'Condition': { 'ArnLike': { - 'AWS:SourceArn': 'rule-arn', + 'AWS:SourceArn': 'arn:aws:events:us-east-1' + ':123456789012:rule/MyScheduledRule', } }, 'Effect': 'Allow', @@ -1929,7 +1931,8 @@ def test_skip_if_permission_already_granted(stubbed_session): stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) awsclient.add_permission_for_cloudwatch_event( - 'rule-arn', 'function-arn') + 'arn:aws:events:us-east-1:123456789012:rule/MyScheduledRule', + 'function-arn') stubbed_session.verify_stubs() @@ -2069,7 +2072,6 @@ def test_add_permission_for_s3_event(stubbed_session): Principal='s3.amazonaws.com', SourceArn='arn:aws:s3:::mybucket', ).returns({}) - stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) @@ -2098,7 +2100,6 @@ def test_skip_if_permission_already_granted_to_s3(stubbed_session): } lambda_client.get_policy( FunctionName='function-arn').returns({'Policy': json.dumps(policy)}) - stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) awsclient.add_permission_for_s3_event( @@ -2172,13 +2173,13 @@ def test_add_permission_for_sns_publish(stubbed_session): FunctionName='function-arn', StatementId=stub.ANY, Principal='sns.amazonaws.com', - SourceArn='arn:aws:sns:::topic-arn', + SourceArn='arn:aws:sns:us-west-2:12345:my-demo-topic', ).returns({}) stubbed_session.activate_stubs() awsclient = TypedAWSClient(stubbed_session) awsclient.add_permission_for_sns_topic( - 'arn:aws:sns:::topic-arn', 'function-arn') + 'arn:aws:sns:us-west-2:12345:my-demo-topic', 'function-arn') stubbed_session.verify_stubs() @@ -2265,7 +2266,7 @@ def test_subscription_not_exists(stubbed_session): def test_can_remove_lambda_sns_permission(stubbed_session): - topic_arn = 'arn:sns:topic' + topic_arn = 'arn:aws:sns:us-west-2:12345:my-demo-topic' policy = { 'Id': 'default', 'Statement': [create_policy_statement(topic_arn, diff --git a/tests/functional/test_package.py b/tests/functional/test_package.py index e9c24c027e..dbc66d077f 100644 --- a/tests/functional/test_package.py +++ b/tests/functional/test_package.py @@ -7,6 +7,7 @@ import pytest import mock +from chalice.awsclient import TypedAWSClient from chalice.config import Config from chalice import Chalice from chalice import package @@ -19,6 +20,7 @@ from chalice.deploy.packager import InvalidSourceDistributionNameError from chalice.compat import pip_no_compile_c_env_vars from chalice.compat import pip_no_compile_c_shim +from chalice.package import PackageOptions from chalice.utils import OSUtils @@ -925,7 +927,7 @@ def test_build_into_existing_dir_with_preinstalled_packages( assert installed_packages == ['bar'] -def test_can_create_app_packager_with_no_autogen(tmpdir): +def test_can_create_app_packager_with_no_autogen(tmpdir, stubbed_session): appdir = _create_app_structure(tmpdir) outdir = tmpdir.mkdir('outdir') @@ -933,7 +935,8 @@ def test_can_create_app_packager_with_no_autogen(tmpdir): config = Config.create(project_dir=str(appdir), chalice_app=sample_app(), **default_params) - p = package.create_app_packager(config) + options = PackageOptions(TypedAWSClient(session=stubbed_session)) + p = package.create_app_packager(config, options) p.package_app(config, str(outdir), 'dev') # We're not concerned with the contents of the files # (those are tested in the unit tests), we just want to make @@ -943,7 +946,7 @@ def test_can_create_app_packager_with_no_autogen(tmpdir): assert 'sam.json' in contents -def test_can_create_app_packager_with_yaml_extention(tmpdir): +def test_can_create_app_packager_with_yaml_extention(tmpdir, stubbed_session): appdir = _create_app_structure(tmpdir) outdir = tmpdir.mkdir('outdir') @@ -953,7 +956,9 @@ def test_can_create_app_packager_with_yaml_extention(tmpdir): config = Config.create(project_dir=str(appdir), chalice_app=sample_app(), **default_params) - p = package.create_app_packager(config, merge_template=str(extras_file)) + options = PackageOptions(TypedAWSClient(session=stubbed_session)) + p = package.create_app_packager(config, options, + merge_template=str(extras_file)) p.package_app(config, str(outdir), 'dev') contents = os.listdir(str(outdir)) @@ -961,7 +966,7 @@ def test_can_create_app_packager_with_yaml_extention(tmpdir): assert 'sam.yaml' in contents -def test_can_specify_yaml_output(tmpdir): +def test_can_specify_yaml_output(tmpdir, stubbed_session): appdir = _create_app_structure(tmpdir) outdir = tmpdir.mkdir('outdir') @@ -969,7 +974,8 @@ def test_can_specify_yaml_output(tmpdir): config = Config.create(project_dir=str(appdir), chalice_app=sample_app(), **default_params) - p = package.create_app_packager(config, template_format='yaml') + options = PackageOptions(TypedAWSClient(session=stubbed_session)) + p = package.create_app_packager(config, options, template_format='yaml') p.package_app(config, str(outdir), 'dev') contents = os.listdir(str(outdir)) @@ -977,14 +983,15 @@ def test_can_specify_yaml_output(tmpdir): assert 'sam.yaml' in contents -def test_will_create_outdir_if_needed(tmpdir): +def test_will_create_outdir_if_needed(tmpdir, stubbed_session): appdir = _create_app_structure(tmpdir) outdir = str(appdir.join('outdir')) default_params = {'autogen_policy': True} config = Config.create(project_dir=str(appdir), chalice_app=sample_app(), **default_params) - p = package.create_app_packager(config) + options = PackageOptions(TypedAWSClient(session=stubbed_session)) + p = package.create_app_packager(config, options) p.package_app(config, str(outdir), 'dev') contents = os.listdir(str(outdir)) assert 'deployment.zip' in contents diff --git a/tests/unit/deploy/test_appgraph.py b/tests/unit/deploy/test_appgraph.py index 548ee9c288..f107405d34 100644 --- a/tests/unit/deploy/test_appgraph.py +++ b/tests/unit/deploy/test_appgraph.py @@ -330,7 +330,7 @@ def test_can_build_private_rest_api(self, sample_app): {'Action': 'execute-api:Invoke', 'Effect': 'Allow', 'Principal': '*', - 'Resource': 'arn:aws:execute-api:*:*:*', + 'Resource': 'arn:*:execute-api:*:*:*', 'Condition': { 'StringEquals': { 'aws:SourceVpce': 'vpce-abc123'}}}, diff --git a/tests/unit/deploy/test_deployer.py b/tests/unit/deploy/test_deployer.py index df6999c9b9..0d58388088 100644 --- a/tests/unit/deploy/test_deployer.py +++ b/tests/unit/deploy/test_deployer.py @@ -836,10 +836,11 @@ def test_templated_swagger_generator(sample_app): uri = doc['paths']['/']['get']['x-amazon-apigateway-integration']['uri'] assert isinstance(uri, StringFormat) assert uri.template == ( - 'arn:aws:apigateway:{region_name}:lambda:path' + 'arn:{partition}:apigateway:{region_name}:lambda:path' '/2015-03-31/functions/{api_handler_lambda_arn}/invocations' ) - assert uri.variables == ['region_name', 'api_handler_lambda_arn'] + assert uri.variables == ['partition', 'region_name', + 'api_handler_lambda_arn'] def test_templated_swagger_with_auth_uri(sample_app_with_auth): @@ -848,10 +849,10 @@ def test_templated_swagger_with_auth_uri(sample_app_with_auth): 'x-amazon-apigateway-authorizer']['authorizerUri'] assert isinstance(uri, StringFormat) assert uri.template == ( - 'arn:aws:apigateway:{region_name}:lambda:path' + 'arn:{partition}:apigateway:{region_name}:lambda:path' '/2015-03-31/functions/{myauth_lambda_arn}/invocations' ) - assert uri.variables == ['region_name', 'myauth_lambda_arn'] + assert uri.variables == ['partition', 'region_name', 'myauth_lambda_arn'] class TestRecordResults(object): diff --git a/tests/unit/deploy/test_executor.py b/tests/unit/deploy/test_executor.py index 809a7652d6..dd5c80275a 100644 --- a/tests/unit/deploy/test_executor.py +++ b/tests/unit/deploy/test_executor.py @@ -16,6 +16,7 @@ class TestExecutor(object): def setup_method(self): self.mock_client = mock.Mock(spec=TypedAWSClient) + self.mock_client.endpoint_dns_suffix.return_value = 'amazonaws.com' self.ui = mock.Mock(spec=UI) self.executor = Executor(self.mock_client, self.ui) @@ -190,9 +191,48 @@ def test_can_call_builtin_function(self): ) ]) assert self.executor.variables['result'] == { + 'partition': 'aws', 'account_id': '123', 'region': 'us-west-2', - 'service': 'lambda' + 'service': 'lambda', + 'dns_suffix': 'amazonaws.com' + } + + def test_built_in_function_interrogate_profile(self): + self.mock_client.region_name = 'us-west-2' + self.mock_client.partition_name = 'aws' + self.execute([ + BuiltinFunction( + function_name='interrogate_profile', + args=[], + output_var='result', + ) + ]) + assert self.executor.variables['result'] == { + 'partition': 'aws', + 'region': 'us-west-2', + 'dns_suffix': 'amazonaws.com' + } + + def test_built_in_function_service_principal(self): + self.mock_client.region_name = 'us-west-2' + self.mock_client.partition_name = 'aws' + self.mock_client.service_principal.return_value = \ + 'apigateway.amazonaws.com' + self.execute([ + BuiltinFunction( + function_name='service_principal', + args=['apigateway'], + output_var='result', + ) + ]) + + self.mock_client.service_principal \ + .assert_called_once_with('apigateway', + 'us-west-2', + 'amazonaws.com') + assert self.executor.variables['result'] == { + 'principal': 'apigateway.amazonaws.com' } def test_errors_out_on_unknown_function(self): diff --git a/tests/unit/deploy/test_planner.py b/tests/unit/deploy/test_planner.py index 5738c83730..26bf688332 100644 --- a/tests/unit/deploy/test_planner.py +++ b/tests/unit/deploy/test_planner.py @@ -127,10 +127,10 @@ def test_can_plan_for_iam_role_creation(self): expected = models.APICall( method_name='create_role', params={'name': 'myrole', - 'trust_policy': {'trust': 'policy'}, + 'trust_policy': Variable('lambda_trust_policy'), 'policy': {'iam': 'policy'}}, ) - self.assert_apicall_equals(plan[0], expected) + self.assert_apicall_equals(plan[4], expected) assert list(self.last_plan.messages.values()) == [ 'Creating IAM role: myrole\n' ] @@ -148,10 +148,10 @@ def test_can_create_plan_for_filebased_role(self): expected = models.APICall( method_name='create_role', params={'name': 'myrole', - 'trust_policy': {'trust': 'policy'}, + 'trust_policy': Variable('lambda_trust_policy'), 'policy': {'iam': 'policy'}}, ) - self.assert_apicall_equals(plan[0], expected) + self.assert_apicall_equals(plan[4], expected) assert list(self.last_plan.messages.values()) == [ 'Creating IAM role: myrole\n' ] @@ -567,7 +567,7 @@ class TestPlanWebsocketAPI(BasePlannerTests): def assert_loads_needed_variables(self, plan): # Parse arn and store region/account id for future # API calls. - assert plan[0:3] == [ + assert plan[0:5] == [ models.BuiltinFunction( 'parse_arn', [Variable('function_name_connect_lambda_arn')], output_var='parsed_lambda_arn', @@ -578,6 +578,12 @@ def assert_loads_needed_variables(self, plan): models.JPSearch('region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch('partition', + input_var='parsed_lambda_arn', + output_var='partition'), + models.JPSearch('dns_suffix', + input_var='parsed_lambda_arn', + output_var='dns_suffix'), ] def test_can_plan_websocket_api(self): @@ -598,7 +604,7 @@ def test_can_plan_websocket_api(self): ) plan = self.determine_plan(websocket_api) self.assert_loads_needed_variables(plan) - assert plan[3:] == [ + assert plan[5:] == [ models.APICall( method_name='create_websocket_api', params={'name': 'app-dev-websocket-api'}, @@ -611,11 +617,11 @@ def test_can_plan_websocket_api(self): models.StoreValue( name='websocket-connect-integration-lambda-path', value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}:lambda' + ':{region_name}:{account_id}:function:%s/' 'invocations' % 'appname-dev-function_name_connect', - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), models.APICall( @@ -631,11 +637,11 @@ def test_can_plan_websocket_api(self): models.StoreValue( name='websocket-message-integration-lambda-path', value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}:lambda' + ':{region_name}:{account_id}:function:%s/' 'invocations' % 'appname-dev-function_name_message', - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), models.APICall( @@ -651,11 +657,11 @@ def test_can_plan_websocket_api(self): models.StoreValue( name='websocket-disconnect-integration-lambda-path', value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}:lambda' + ':{region_name}:{account_id}:function:%s/' 'invocations' % 'appname-dev-function_name_disconnect', - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), models.APICall( @@ -711,8 +717,8 @@ def test_can_plan_websocket_api(self): name='websocket_api_url', value=StringFormat( 'wss://{websocket_api_id}.execute-api.{region_name}' - '.amazonaws.com/%s/' % 'api', - ['websocket_api_id', 'region_name'], + '.{dns_suffix}/%s/' % 'api', + ['websocket_api_id', 'region_name', 'dns_suffix'], ), ), models.RecordResourceVariable( @@ -773,7 +779,7 @@ def test_can_update_websocket_api(self): } plan = self.determine_plan(websocket_api) self.assert_loads_needed_variables(plan) - assert plan[3:] == [ + assert plan[5:] == [ models.StoreValue( name='websocket_api_id', value='my_websocket_api_id', @@ -801,11 +807,11 @@ def test_can_update_websocket_api(self): models.StoreValue( name='websocket-connect-integration-lambda-path', value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}:lambda' + ':{region_name}:{account_id}:function:%s/' 'invocations' % 'appname-dev-function_name_connect', - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), models.APICall( @@ -821,11 +827,11 @@ def test_can_update_websocket_api(self): models.StoreValue( name='websocket-message-integration-lambda-path', value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}:lambda' + ':{region_name}:{account_id}:function:%s/' 'invocations' % 'appname-dev-function_name_message', - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), models.APICall( @@ -841,11 +847,11 @@ def test_can_update_websocket_api(self): models.StoreValue( name='websocket-disconnect-integration-lambda-path', value=StringFormat( - 'arn:aws:apigateway:{region_name}:lambda:path/' - '2015-03-31/functions/arn:aws:lambda:{region_name}:' - '{account_id}:function:%s/' + 'arn:{partition}:apigateway:{region_name}:lambda:path/' + '2015-03-31/functions/arn:{partition}:lambda' + ':{region_name}:{account_id}:function:%s/' 'invocations' % 'appname-dev-function_name_disconnect', - ['region_name', 'account_id'], + ['partition', 'region_name', 'account_id'], ), ), models.APICall( @@ -886,8 +892,8 @@ def test_can_update_websocket_api(self): name='websocket_api_url', value=StringFormat( 'wss://{websocket_api_id}.execute-api.{region_name}' - '.amazonaws.com/%s/' % 'api', - ['websocket_api_id', 'region_name'], + '.{dns_suffix}/%s/' % 'api', + ['websocket_api_id', 'region_name', 'dns_suffix'], ), ), models.RecordResourceVariable( @@ -932,7 +938,7 @@ class TestPlanRestAPI(BasePlannerTests): def assert_loads_needed_variables(self, plan): # Parse arn and store region/account id for future # API calls. - assert plan[0:4] == [ + assert plan[0:6] == [ models.BuiltinFunction( 'parse_arn', [Variable('function_name_lambda_arn')], output_var='parsed_lambda_arn', @@ -943,6 +949,12 @@ def assert_loads_needed_variables(self, plan): models.JPSearch('region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch('partition', + input_var='parsed_lambda_arn', + output_var='partition'), + models.JPSearch('dns_suffix', + input_var='parsed_lambda_arn', + output_var='dns_suffix'), # Verify we copy the function arn as needed. models.CopyVariable( from_var='function_name_lambda_arn', @@ -962,7 +974,7 @@ def test_can_plan_rest_api(self): plan = self.determine_plan(rest_api) self.assert_loads_needed_variables(plan) - assert plan[4:] == [ + assert plan[6:] == [ models.APICall( method_name='import_rest_api', params={'swagger_document': {'swagger': '2.0'}, @@ -1002,8 +1014,8 @@ def test_can_plan_rest_api(self): name='rest_api_url', value=StringFormat( 'https://{rest_api_id}.execute-api.{region_name}' - '.amazonaws.com/api/', - ['rest_api_id', 'region_name'], + '.{dns_suffix}/api/', + ['rest_api_id', 'region_name', 'dns_suffix'], ), ), models.RecordResourceVariable( @@ -1034,7 +1046,7 @@ def test_can_update_rest_api_with_policy(self): } plan = self.determine_plan(rest_api) - assert plan[8].params == { + assert plan[10].params == { 'patch_operations': [ {'op': 'replace', 'path': '/minimumCompressionSize', @@ -1066,7 +1078,7 @@ def test_can_update_rest_api(self): plan = self.determine_plan(rest_api) self.assert_loads_needed_variables(plan) - assert plan[4:] == [ + assert plan[6:] == [ models.StoreValue(name='rest_api_id', value='my_rest_api_id'), models.RecordResourceVariable( resource_type='rest_api', @@ -1119,8 +1131,8 @@ def test_can_update_rest_api(self): name='rest_api_url', value=StringFormat( 'https://{rest_api_id}.execute-api.{region_name}' - '.amazonaws.com/api/', - ['rest_api_id', 'region_name'], + '.{dns_suffix}/api/', + ['rest_api_id', 'region_name', 'dns_suffix'], ), ), models.RecordResourceVariable( @@ -1141,7 +1153,7 @@ def test_can_plan_sns_subscription(self): lambda_function=function ) plan = self.determine_plan(sns_subscription) - plan_parse_arn = plan[:4] + plan_parse_arn = plan[:5] assert plan_parse_arn == [ models.BuiltinFunction( function_name='parse_arn', @@ -1155,16 +1167,20 @@ def test_can_plan_sns_subscription(self): expression='region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch( + expression='partition', + input_var='parsed_lambda_arn', + output_var='partition'), models.StoreValue( name='function_name-sns-subscription_topic_arn', value=StringFormat( - "arn:aws:sns:{region_name}:{account_id}:mytopic", - variables=['region_name', 'account_id'], + "arn:{partition}:sns:{region_name}:{account_id}:mytopic", + variables=['partition', 'region_name', 'account_id'], ) ), ] topic_arn_var = Variable("function_name-sns-subscription_topic_arn") - assert plan[4:] == [ + assert plan[5:] == [ models.APICall( method_name='add_permission_for_sns_topic', params={ @@ -1278,7 +1294,7 @@ def test_sns_subscription_exists_is_noop_for_planner(self): subscription_arn='arn:aws:subscribe', ) plan = self.determine_plan(sns_subscription) - plan_parse_arn = plan[:4] + plan_parse_arn = plan[:5] assert plan_parse_arn == [ models.BuiltinFunction( function_name='parse_arn', @@ -1292,15 +1308,19 @@ def test_sns_subscription_exists_is_noop_for_planner(self): expression='region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch( + expression='partition', + input_var='parsed_lambda_arn', + output_var='partition'), models.StoreValue( name='function_name-sns-subscription_topic_arn', value=StringFormat( - "arn:aws:sns:{region_name}:{account_id}:mytopic", - variables=['region_name', 'account_id'], + "arn:{partition}:sns:{region_name}:{account_id}:mytopic", + variables=['partition', 'region_name', 'account_id'], ) ), ] - assert plan[4:] == [ + assert plan[5:] == [ models.RecordResourceValue( resource_type='sns_event', resource_name='function_name-sns-subscription', @@ -1337,7 +1357,7 @@ def test_can_plan_sqs_event_source(self): lambda_function=function ) plan = self.determine_plan(sqs_event_source) - plan_parse_arn = plan[:4] + plan_parse_arn = plan[:5] assert plan_parse_arn == [ models.BuiltinFunction( function_name='parse_arn', @@ -1354,15 +1374,20 @@ def test_can_plan_sqs_event_source(self): input_var='parsed_lambda_arn', output_var='region_name' ), + models.JPSearch( + expression='partition', + input_var='parsed_lambda_arn', + output_var='partition' + ), models.StoreValue( name='function_name-sqs-event-source_queue_arn', value=StringFormat( - "arn:aws:sqs:{region_name}:{account_id}:myqueue", - variables=['region_name', 'account_id'], + "arn:{partition}:sqs:{region_name}:{account_id}:myqueue", + variables=['partition', 'region_name', 'account_id'], ), ) ] - assert plan[4:] == [ + assert plan[5:] == [ models.APICall( method_name='create_sqs_event_source', params={ @@ -1417,7 +1442,7 @@ def test_sqs_event_source_exists_updates_batch_size(self): event_uuid='my-uuid', ) plan = self.determine_plan(sqs_event_source) - plan_parse_arn = plan[:4] + plan_parse_arn = plan[:5] assert plan_parse_arn == [ models.BuiltinFunction( function_name='parse_arn', @@ -1431,15 +1456,20 @@ def test_sqs_event_source_exists_updates_batch_size(self): expression='region', input_var='parsed_lambda_arn', output_var='region_name'), + models.JPSearch( + expression='partition', + input_var='parsed_lambda_arn', + output_var='partition' + ), models.StoreValue( name='function_name-sqs-event-source_queue_arn', value=StringFormat( - "arn:aws:sqs:{region_name}:{account_id}:myqueue", - variables=['region_name', 'account_id'], + "arn:{partition}:sqs:{region_name}:{account_id}:myqueue", + variables=['partition', 'region_name', 'account_id'], ), ) ] - assert plan[4:] == [ + assert plan[5:] == [ models.APICall( method_name='update_sqs_event_source', params={ diff --git a/tests/unit/deploy/test_swagger.py b/tests/unit/deploy/test_swagger.py index 6e84e4b98b..ed9e579101 100644 --- a/tests/unit/deploy/test_swagger.py +++ b/tests/unit/deploy/test_swagger.py @@ -12,7 +12,10 @@ def swagger_gen(): return SwaggerGenerator( region='us-west-2', - deployed_resources={'api_handler_arn': 'lambda_arn'}) + deployed_resources={ + 'api_handler_arn': 'arn:aws:lambda:mars-west-1:123456789' + ':function:lambda_arn' + }) def test_can_add_binary_media_types(swagger_gen): @@ -141,7 +144,8 @@ def test_apigateway_integration_generation(sample_app, swagger_gen): assert apig_integ['type'] == 'aws_proxy' assert apig_integ['uri'] == ( "arn:aws:apigateway:us-west-2:lambda:path" - "/2015-03-31/functions/lambda_arn/invocations" + "/2015-03-31/functions/" + "arn:aws:lambda:mars-west-1:123456789:function:lambda_arn/invocations" ) assert 'responses' in apig_integ responses = apig_integ['responses'] @@ -513,11 +517,13 @@ def test_builtin_auth(sample_app): swagger_gen = SwaggerGenerator( region='us-west-2', deployed_resources={ - 'api_handler_arn': 'lambda_arn', + 'api_handler_arn': 'arn:aws:lambda:mars-west-1:123456789' + ':function:lambda_arn', 'api_handler_name': 'api-dev', 'lambda_functions': { 'api-dev-myauth': { - 'arn': 'auth_arn', + 'arn': 'arn:aws:lambda:mars-west-1:123456789' + ':function:auth_arn', 'type': 'authorizer', } } @@ -546,7 +552,9 @@ def foo(): 'authorizerCredentials': 'arn:role', 'authorizerResultTtlInSeconds': 10, 'authorizerUri': ('arn:aws:apigateway:us-west-2:lambda:path' - '/2015-03-31/functions/auth_arn/invocations'), + '/2015-03-31/functions/' + 'arn:aws:lambda:mars-west-1:123456789:function' + ':auth_arn/invocations'), } } @@ -555,11 +563,13 @@ def test_will_default_to_function_name_for_auth(sample_app): swagger_gen = SwaggerGenerator( region='us-west-2', deployed_resources={ - 'api_handler_arn': 'lambda_arn', + 'api_handler_arn': 'arn:aws:lambda:mars-west-1:123456789' + ':function:lambda_arn', 'api_handler_name': 'api-dev', 'lambda_functions': { 'api-dev-auth': { - 'arn': 'auth_arn', + 'arn': 'arn:aws:lambda:mars-west-1:123456789' + ':function:auth_arn', 'type': 'authorizer', } } @@ -588,7 +598,9 @@ def foo(): 'authorizerCredentials': 'arn:role', 'authorizerResultTtlInSeconds': 10, 'authorizerUri': ('arn:aws:apigateway:us-west-2:lambda:path' - '/2015-03-31/functions/auth_arn/invocations'), + '/2015-03-31/functions/' + 'arn:aws:lambda:mars-west-1:123456789:function' + ':auth_arn/invocations'), } } @@ -697,8 +709,9 @@ def foo(): 'authorizerResultTtlInSeconds': 10, 'authorizerUri': { 'Fn::Sub': ( - 'arn:aws:apigateway:${AWS::Region}:lambda:path' - '/2015-03-31/functions/${Auth.Arn}/invocations' + 'arn:${AWS::Partition}:apigateway:${AWS::Region}' + ':lambda:path/2015-03-31/functions/' + '${Auth.Arn}/invocations' ) } } diff --git a/tests/unit/test_awsclient.py b/tests/unit/test_awsclient.py new file mode 100644 index 0000000000..96e0063593 --- /dev/null +++ b/tests/unit/test_awsclient.py @@ -0,0 +1,162 @@ +from collections import OrderedDict + +import pytest + +from chalice.awsclient import TypedAWSClient + + +@pytest.mark.parametrize('service,region,endpoint', [ + ('sns', 'us-east-1', + OrderedDict([('partition', 'aws'), + ('endpointName', 'us-east-1'), + ('protocols', ['http', 'https']), + ('hostname', 'sns.us-east-1.amazonaws.com'), + ('signatureVersions', ['v4']), + ('dnsSuffix', 'amazonaws.com')])), + ('sqs', 'cn-north-1', + OrderedDict([('partition', 'aws-cn'), + ('endpointName', 'cn-north-1'), + ('protocols', ['http', 'https']), + ('sslCommonName', 'cn-north-1.queue.amazonaws.com.cn'), + ('hostname', 'sqs.cn-north-1.amazonaws.com.cn'), + ('signatureVersions', ['v4']), + ('dnsSuffix', 'amazonaws.com.cn')])), + ('dynamodb', 'mars-west-1', None) +]) +def test_resolve_endpoint(stubbed_session, service, region, endpoint): + awsclient = TypedAWSClient(stubbed_session) + assert endpoint == awsclient.resolve_endpoint(service, region) + + +@pytest.mark.parametrize('arn,endpoint', [ + ('arn:aws:sns:us-east-1:123456:MyTopic', + OrderedDict([('partition', 'aws'), + ('endpointName', 'us-east-1'), + ('protocols', ['http', 'https']), + ('hostname', 'sns.us-east-1.amazonaws.com'), + ('signatureVersions', ['v4']), + ('dnsSuffix', 'amazonaws.com')])), + ('arn:aws-cn:sqs:cn-north-1:444455556666:queue1', + OrderedDict([('partition', 'aws-cn'), + ('endpointName', 'cn-north-1'), + ('protocols', ['http', 'https']), + ('sslCommonName', 'cn-north-1.queue.amazonaws.com.cn'), + ('hostname', 'sqs.cn-north-1.amazonaws.com.cn'), + ('signatureVersions', ['v4']), + ('dnsSuffix', 'amazonaws.com.cn')])), + ('arn:aws:dynamodb:mars-west-1:123456:table/MyTable', None) +]) +def test_endpoint_from_arn(stubbed_session, arn, endpoint): + awsclient = TypedAWSClient(stubbed_session) + assert endpoint == awsclient.endpoint_from_arn(arn) + + +@pytest.mark.parametrize('service,region,dns_suffix', [ + ('sns', 'us-east-1', 'amazonaws.com'), + ('sns', 'cn-north-1', 'amazonaws.com.cn'), + ('dynamodb', 'mars-west-1', 'amazonaws.com') +]) +def test_endpoint_dns_suffix(stubbed_session, service, region, dns_suffix): + awsclient = TypedAWSClient(stubbed_session) + assert dns_suffix == awsclient.endpoint_dns_suffix(service, region) + + +@pytest.mark.parametrize('arn,dns_suffix', [ + ('arn:aws:sns:us-east-1:123456:MyTopic', 'amazonaws.com'), + ('arn:aws-cn:sqs:cn-north-1:444455556666:queue1', 'amazonaws.com.cn'), + ('arn:aws:dynamodb:mars-west-1:123456:table/MyTable', 'amazonaws.com') +]) +def test_endpoint_dns_suffix_from_arn(stubbed_session, arn, dns_suffix): + awsclient = TypedAWSClient(stubbed_session) + assert dns_suffix == awsclient.endpoint_dns_suffix_from_arn(arn) + + +class TestServicePrincipal(object): + + @pytest.fixture + def region(self): + return 'bermuda-triangle-42' + + @pytest.fixture + def url_suffix(self): + return '.nowhere.null' + + @pytest.fixture + def non_iso_suffixes(self): + return ['', '.amazonaws.com', '.amazonaws.com.cn'] + + @pytest.fixture + def awsclient(self, stubbed_session): + return TypedAWSClient(stubbed_session) + + def test_unmatched_service(self, awsclient): + assert awsclient.service_principal('taco.magic.food.com', + 'us-east-1', + 'amazonaws.com') == \ + 'taco.magic.food.com' + + def test_defaults(self, awsclient): + assert awsclient.service_principal('lambda') == 'lambda.amazonaws.com' + + def test_states(self, awsclient, region, url_suffix, non_iso_suffixes): + services = ['states'] + for suffix in non_iso_suffixes: + for service in services: + assert awsclient.service_principal('{}{}'.format(service, + suffix), + region, url_suffix) == \ + '{}.{}.amazonaws.com'.format(service, region) + + def test_codedeploy_and_logs(self, awsclient, region, url_suffix, + non_iso_suffixes): + services = ['codedeploy', 'logs'] + for suffix in non_iso_suffixes: + for service in services: + assert awsclient.service_principal('{}{}'.format(service, + suffix), + region, url_suffix) == \ + '{}.{}.{}'.format(service, region, url_suffix) + + def test_ec2(self, awsclient, region, url_suffix, non_iso_suffixes): + services = ['ec2'] + for suffix in non_iso_suffixes: + for service in services: + assert awsclient.service_principal('{}{}'.format(service, + suffix), + region, url_suffix) == \ + '{}.{}'.format(service, url_suffix) + + def test_others(self, awsclient, region, url_suffix, non_iso_suffixes): + services = ['autoscaling', 'lambda', 'events', 'sns', 'sqs', + 'foo-service'] + for suffix in non_iso_suffixes: + for service in services: + assert awsclient.service_principal('{}{}'.format(service, + suffix), + region, url_suffix) == \ + '{}.amazonaws.com'.format(service) + + def test_local_suffix(self, awsclient, region, url_suffix): + assert awsclient.service_principal('foo-service.local', + region, + url_suffix) == 'foo-service.local' + + def test_states_iso(self, awsclient): + assert awsclient.service_principal('states.amazonaws.com', + 'us-iso-east-1', + 'c2s.ic.gov') == \ + 'states.amazonaws.com' + + def test_states_isob(self, awsclient): + assert awsclient.service_principal('states.amazonaws.com', + 'us-isob-east-1', + 'sc2s.sgov.gov') == \ + 'states.amazonaws.com' + + def test_iso_exceptions(self, awsclient): + services = ['cloudhsm', 'config', 'workspaces'] + for service in services: + assert awsclient.service_principal( + '{}.amazonaws.com'.format(service), + 'us-iso-east-1', + 'c2s.ic.gov') == '{}.c2s.ic.gov'.format(service) diff --git a/tests/unit/test_package.py b/tests/unit/test_package.py index 6b6d9a1ccd..39dcb775e4 100644 --- a/tests/unit/test_package.py +++ b/tests/unit/test_package.py @@ -5,11 +5,13 @@ import pytest from chalice.config import Config from chalice import package +from chalice.constants import LAMBDA_TRUST_POLICY from chalice.deploy.appgraph import ApplicationGraphBuilder, DependencyBuilder +from chalice.awsclient import TypedAWSClient from chalice.deploy.deployer import BuildStage from chalice.deploy import models from chalice.deploy.swagger import SwaggerGenerator -from chalice.constants import LAMBDA_TRUST_POLICY +from chalice.package import PackageOptions from chalice.utils import OSUtils @@ -20,13 +22,15 @@ def mock_swagger_generator(): def test_can_create_app_packager(): config = Config() - packager = package.create_app_packager(config) + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + packager = package.create_app_packager(config, options) assert isinstance(packager, package.AppPackager) def test_can_create_terraform_app_packager(): config = Config() - packager = package.create_app_packager(config, 'terraform') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + packager = package.create_app_packager(config, options, 'terraform') assert isinstance(packager, package.AppPackager) @@ -86,7 +90,10 @@ def test_terraform_post_processor_moves_files_once(): def test_template_generator_default(): - tgen = package.TemplateGenerator(Config()) + tgen = package.TemplateGenerator(Config(), + PackageOptions( + mock.Mock(spec=TypedAWSClient) + )) with pytest.raises(package.UnsupportedFeatureError): tgen.dispatch(models.Model(), {}) @@ -261,18 +268,23 @@ class TemplateTestBase(object): template_gen_factory = None - def setup_method(self): + def setup_method(self, stubbed_session): self.resource_builder = package.ResourceBuilder( application_builder=ApplicationGraphBuilder(), deps_builder=DependencyBuilder(), build_stage=mock.Mock(spec=BuildStage) ) - self.template_gen = self.template_gen_factory(Config()) - - def generate_template(self, config, chalice_stage_name): + client = TypedAWSClient(None) + m_client = mock.Mock(wraps=client, spec=TypedAWSClient) + type(m_client).region_name = mock.PropertyMock( + return_value='us-west-2') + options = PackageOptions(m_client) + self.template_gen = self.template_gen_factory(Config(), options) + + def generate_template(self, config, options, chalice_stage_name): resources = self.resource_builder.construct_resources( config, chalice_stage_name) - return self.template_gen_factory(config).generate(resources) + return self.template_gen_factory(config, options).generate(resources) def lambda_function(self): return models.LambdaFunction( @@ -293,6 +305,24 @@ def lambda_function(self): ) +class TestPackageOptions(object): + + def test_service_principal(self): + awsclient = mock.Mock(spec=TypedAWSClient) + awsclient.region_name = 'us-east-1' + awsclient.endpoint_dns_suffix.return_value = 'amazonaws.com' + awsclient.service_principal.return_value = 'lambda.amazonaws.com' + options = package.PackageOptions(awsclient) + principal = options.service_principal('lambda') + assert principal == 'lambda.amazonaws.com' + + awsclient.endpoint_dns_suffix.assert_called_once_with('lambda', + 'us-east-1') + awsclient.service_principal.assert_called_once_with('lambda', + 'us-east-1', + 'amazonaws.com') + + class TestTerraformTemplate(TemplateTestBase): template_gen_factory = package.TerraformGenerator @@ -306,7 +336,7 @@ class TestTerraformTemplate(TemplateTestBase): } } - def generate_template(self, config, chalice_stage_name): + def generate_template(self, config, options, chalice_stage_name): resources = self.resource_builder.construct_resources( config, 'dev') @@ -330,7 +360,7 @@ def generate_template(self, config, chalice_stage_name): elif isinstance(r, models.FileBasedIAMPolicy): r.document = self.EmptyPolicy - return self.template_gen_factory(config).generate(resources) + return self.template_gen_factory(config, options).generate(resources) def get_function(self, template): functions = list(template['resource'][ @@ -430,7 +460,8 @@ def test_can_generate_scheduled_event(self): 'description': 'description', } - def test_can_generate_rest_api(self, sample_app_with_auth): + def test_can_generate_rest_api(self, sample_app_with_auth, + stubbed_session): config = Config.create(chalice_app=sample_app_with_auth, project_dir='.', minimum_compression_size=8192, @@ -438,7 +469,8 @@ def test_can_generate_rest_api(self, sample_app_with_auth): api_gateway_endpoint_vpce='vpce-abc123', app_name='sample_app', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') resources = template['resource'] # Lambda function should be created. assert resources['aws_lambda_function'] @@ -459,13 +491,13 @@ def test_can_generate_rest_api(self, sample_app_with_auth): 'Statement': [ { 'Action': 'execute-api:Invoke', - 'Resource': 'arn:aws:execute-api:*:*:*', + 'Resource': 'arn:*:execute-api:*:*:*', 'Effect': 'Allow', 'Condition': { 'StringEquals': { 'aws:SourceVpce': 'vpce-abc123' - } - }, + } + }, 'Principal': '*' } ] @@ -503,7 +535,8 @@ def test_can_generate_rest_api(self, sample_app_with_auth): 'value': '${aws_api_gateway_deployment.rest_api.invoke_url}'} } - def test_can_package_s3_event_handler_with_tf_ref(self, sample_app): + def test_can_package_s3_event_handler_with_tf_ref(self, sample_app, + stubbed_session): @sample_app.on_s3_event( bucket='${aws_s3_bucket.my_data_bucket.id}') def handler(event): @@ -512,31 +545,33 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['resource']['aws_s3_bucket_notification'][ - 'my_data_bucket_notify'] == { - 'bucket': '${aws_s3_bucket.my_data_bucket.id}', - 'lambda_function': [{ - 'events': ['s3:ObjectCreated:*'], - 'lambda_function_arn': ( - '${aws_lambda_function.handler.arn}') - }] - } - - def test_can_generate_chalice_terraform_static_data(self, sample_app): + 'my_data_bucket_notify'] == { + 'bucket': '${aws_s3_bucket.my_data_bucket.id}', + 'lambda_function': [{ + 'events': ['s3:ObjectCreated:*'], + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + }] + } + + def test_can_generate_chalice_terraform_static_data(self, sample_app, + stubbed_session): config = Config.create(chalice_app=sample_app, project_dir='.', app_name='myfoo', api_gateway_stage='dev') - - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['data']['null_data_source']['chalice']['inputs'] == { 'app': 'myfoo', 'stage': 'dev' } - def test_can_package_s3_event_handler_sans_filters(self, sample_app): + def test_can_package_s3_event_handler_sans_filters(self, sample_app, + stubbed_session): @sample_app.on_s3_event(bucket='foo') def handler(event): pass @@ -544,19 +579,19 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['resource']['aws_s3_bucket_notification'][ - 'foo_notify'] == { - 'bucket': 'foo', - 'lambda_function': [{ - 'events': ['s3:ObjectCreated:*'], - 'lambda_function_arn': ( - '${aws_lambda_function.handler.arn}') - }] - } - - def test_can_package_s3_event_handler(self, sample_app): + 'foo_notify'] == { + 'bucket': 'foo', + 'lambda_function': [{ + 'events': ['s3:ObjectCreated:*'], + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + }] + } + + def test_can_package_s3_event_handler(self, sample_app, stubbed_session): @sample_app.on_s3_event( bucket='foo', prefix='incoming', suffix='.csv') def handler(event): @@ -566,30 +601,30 @@ def handler(event): project_dir='.', app_name='sample_app', api_gateway_stage='api') - - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['resource']['aws_lambda_permission'][ - 'handler-s3event'] == { - 'action': 'lambda:InvokeFunction', - 'function_name': 'sample_app-dev-handler', - 'principal': 's3.amazonaws.com', - 'source_arn': 'arn:aws:s3:::foo', - 'statement_id': 'handler-s3event' - } + 'handler-s3event'] == { + 'action': 'lambda:InvokeFunction', + 'function_name': 'sample_app-dev-handler', + 'principal': 's3.amazonaws.com', + 'source_arn': 'arn:*:s3:::foo', + 'statement_id': 'handler-s3event' + } assert template['resource']['aws_s3_bucket_notification'][ - 'foo_notify'] == { - 'bucket': 'foo', - 'lambda_function': [{ - 'events': ['s3:ObjectCreated:*'], - 'filter_prefix': 'incoming', - 'filter_suffix': '.csv', - 'lambda_function_arn': ( - '${aws_lambda_function.handler.arn}') - }] - } - - def test_can_package_sns_handler(self, sample_app): + 'foo_notify'] == { + 'bucket': 'foo', + 'lambda_function': [{ + 'events': ['s3:ObjectCreated:*'], + 'filter_prefix': 'incoming', + 'filter_suffix': '.csv', + 'lambda_function_arn': ( + '${aws_lambda_function.handler.arn}') + }] + } + + def test_can_package_sns_handler(self, sample_app, stubbed_session): @sample_app.on_sns_message(topic='foo') def handler(event): pass @@ -597,18 +632,20 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['resource']['aws_sns_topic_subscription'][ - 'handler-sns-subscription'] == { - 'topic_arn': ( - 'arn:aws:sns:${data.aws_region.chalice.name}:' - '${data.aws_caller_identity.chalice.account_id}:foo'), - 'protocol': 'lambda', - 'endpoint': '${aws_lambda_function.handler.arn}' - } - - def test_can_package_sns_arn_handler(self, sample_app): + 'handler-sns-subscription'] == { + 'topic_arn': ( + 'arn:${data.aws_partition.chalice.partition}:sns' + ':${data.aws_region.chalice.name}:' + '${data.aws_caller_identity.chalice.account_id}:foo'), + 'protocol': 'lambda', + 'endpoint': '${aws_lambda_function.handler.arn}' + } + + def test_can_package_sns_arn_handler(self, sample_app, stubbed_session): arn = 'arn:aws:sns:space-leo-1:1234567890:foo' @sample_app.on_sns_message(topic=arn) @@ -619,24 +656,25 @@ def handler(event): project_dir='.', app_name='sample_app', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['resource']['aws_sns_topic_subscription'][ - 'handler-sns-subscription'] == { - 'topic_arn': arn, - 'protocol': 'lambda', - 'endpoint': '${aws_lambda_function.handler.arn}' - } + 'handler-sns-subscription'] == { + 'topic_arn': arn, + 'protocol': 'lambda', + 'endpoint': '${aws_lambda_function.handler.arn}' + } assert template['resource']['aws_lambda_permission'][ - 'handler-sns-subscription'] == { - 'function_name': 'sample_app-dev-handler', - 'action': 'lambda:InvokeFunction', - 'principal': 'sns.amazonaws.com', - 'source_arn': 'arn:aws:sns:space-leo-1:1234567890:foo' - } - - def test_can_package_sqs_handler(self, sample_app): + 'handler-sns-subscription'] == { + 'function_name': 'sample_app-dev-handler', + 'action': 'lambda:InvokeFunction', + 'principal': 'sns.amazonaws.com', + 'source_arn': 'arn:aws:sns:space-leo-1:1234567890:foo' + } + + def test_can_package_sqs_handler(self, sample_app, stubbed_session): @sample_app.on_sqs_message(queue='foo', batch_size=5) def handler(event): pass @@ -645,25 +683,29 @@ def handler(event): project_dir='.', app_name='sample_app', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') assert template['resource'][ - 'aws_lambda_event_source_mapping'][ - 'handler-sqs-event-source'] == { - 'event_source_arn': ( - 'arn:aws:sqs:${data.aws_region.chalice.name}:' - '${data.aws_caller_identity.chalice.account_id}:foo'), - 'function_name': 'sample_app-dev-handler', - 'batch_size': 5 - } - - def test_package_websocket_with_error_message(self, sample_websocket_app): + 'aws_lambda_event_source_mapping'][ + 'handler-sqs-event-source'] == { + 'event_source_arn': ( + 'arn:${data.aws_partition.chalice.partition}:sqs' + ':${data.aws_region.chalice.name}:' + '${data.aws_caller_identity.chalice.account_id}:foo'), + 'function_name': 'sample_app-dev-handler', + 'batch_size': 5 + } + + def test_package_websocket_with_error_message(self, sample_websocket_app, + stubbed_session): config = Config.create(chalice_app=sample_websocket_app, project_dir='.', app_name='sample_app', api_gateway_stage='api') + options = PackageOptions(TypedAWSClient(stubbed_session)) with pytest.raises(NotImplementedError) as excinfo: - self.generate_template(config, 'dev') + self.generate_template(config, options, 'dev') # Should mention the decorator name. assert 'Websocket decorators' in str(excinfo.value) @@ -679,7 +721,8 @@ def test_sam_generates_sam_template_basic(self, sample_app): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') # Verify the basic structure is in place. The specific parts # are validated in other tests. assert template['AWSTemplateFormatVersion'] == '2010-09-09' @@ -878,18 +921,21 @@ def test_can_generate_rest_api_without_compression( project_dir='.', api_gateway_stage='api', ) - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') resources = template['Resources'] assert 'MinimumCompressionSize' not in \ resources['RestAPI']['Properties'] - def test_can_generate_rest_api(self, sample_app_with_auth): + def test_can_generate_rest_api(self, sample_app_with_auth, + stubbed_session): config = Config.create(chalice_app=sample_app_with_auth, project_dir='.', api_gateway_stage='api', minimum_compression_size=100, ) - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') resources = template['Resources'] # Lambda function should be created. assert resources['APIHandler']['Type'] == 'AWS::Serverless::Function' @@ -902,8 +948,8 @@ def test_can_generate_rest_api(self, sample_app_with_auth): 'Principal': 'apigateway.amazonaws.com', 'SourceArn': { 'Fn::Sub': [ - ('arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}' - ':${RestAPIId}/*'), + ('arn:${AWS::Partition}:execute-api:${AWS::Region}' + ':${AWS::AccountId}:${RestAPIId}/*'), {'RestAPIId': {'Ref': 'RestAPI'}}]}}, } assert resources['RestAPI']['Type'] == 'AWS::Serverless::Api' @@ -920,8 +966,8 @@ def test_can_generate_rest_api(self, sample_app_with_auth): 'Principal': 'apigateway.amazonaws.com', 'SourceArn': { 'Fn::Sub': [ - ('arn:aws:execute-api:${AWS::Region}:${AWS::AccountId}' - ':${RestAPIId}/*'), + ('arn:${AWS::Partition}:execute-api:${AWS::Region}' + ':${AWS::AccountId}:${RestAPIId}/*'), {'RestAPIId': {'Ref': 'RestAPI'}}]}}, } # Also verify we add the expected outputs when using @@ -937,7 +983,7 @@ def test_can_generate_rest_api(self, sample_app_with_auth): 'Value': { 'Fn::Sub': ( 'https://${RestAPI}.execute-api.' - '${AWS::Region}.amazonaws.com/api/' + '${AWS::Region}.${AWS::URLSuffix}/api/' ) } }, @@ -960,18 +1006,21 @@ def test_generate_partial_websocket_api( config = Config.create(chalice_app=sample_websocket_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') resources = template['Resources'] # Check that the template's deployment only depends on the one route. depends_on = resources['WebsocketAPIDeployment'].pop('DependsOn') assert [route] == depends_on - def test_generate_websocket_api(self, sample_websocket_app): + def test_generate_websocket_api(self, sample_websocket_app, + stubbed_session): config = Config.create(chalice_app=sample_websocket_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(TypedAWSClient(stubbed_session)) + template = self.generate_template(config, options, 'dev') resources = template['Resources'] assert resources['WebsocketAPI']['Type'] == 'AWS::ApiGatewayV2::Api' @@ -980,8 +1029,7 @@ def test_generate_websocket_api(self, sample_websocket_app): ('WebsocketMessage', '$default'), ('WebsocketDisconnect', '$disconnect'),): # Lambda function should be created. - assert resources[handler][ - 'Type'] == 'AWS::Serverless::Function' + assert resources[handler]['Type'] == 'AWS::Serverless::Function' # Along with permission to invoke from API Gateway. assert resources['%sInvokePermission' % handler] == { @@ -993,7 +1041,8 @@ def test_generate_websocket_api(self, sample_websocket_app): 'SourceArn': { 'Fn::Sub': [ ( - 'arn:aws:execute-api:${AWS::Region}:${AWS::' + 'arn:${AWS::Partition}:execute-api' + ':${AWS::Region}:${AWS::' 'AccountId}:${WebsocketAPIId}/*' ), {'WebsocketAPIId': {'Ref': 'WebsocketAPI'}}]}}, @@ -1012,10 +1061,11 @@ def test_generate_websocket_api(self, sample_websocket_app): 'IntegrationUri': { 'Fn::Sub': [ ( - 'arn:aws:apigateway:${AWS::Region}:lambda:path' - '/2015-03-31/functions/arn:aws:lambda:' - '${AWS::Region}:' '${AWS::AccountId}:function:' - '${WebsocketHandler}/invocations' + 'arn:${AWS::Partition}:apigateway' + ':${AWS::Region}:lambda:path' + '/2015-03-31/functions/arn:${AWS::Partition}' + ':lambda:${AWS::Region}:${AWS::AccountId}' + ':function:${WebsocketHandler}/invocations' ), {'WebsocketHandler': {'Ref': handler}} ], @@ -1079,15 +1129,15 @@ def test_generate_websocket_api(self, sample_websocket_app): 'Fn::GetAtt': ['WebsocketConnect', 'Arn'] } }, - 'WebsocketConnectHandlerName': {'Value': {'Ref': - 'WebsocketConnect'}}, + 'WebsocketConnectHandlerName': { + 'Value': {'Ref': 'WebsocketConnect'}}, 'WebsocketMessageHandlerArn': { 'Value': { 'Fn::GetAtt': ['WebsocketMessage', 'Arn'] } }, - 'WebsocketMessageHandlerName': {'Value': {'Ref': - 'WebsocketMessage'}}, + 'WebsocketMessageHandlerName': { + 'Value': {'Ref': 'WebsocketMessage'}}, 'WebsocketDisconnectHandlerArn': { 'Value': { 'Fn::GetAtt': ['WebsocketDisconnect', 'Arn'] @@ -1099,7 +1149,7 @@ def test_generate_websocket_api(self, sample_websocket_app): 'Value': { 'Fn::Sub': ( 'wss://${WebsocketAPI}.execute-api.' - '${AWS::Region}.amazonaws.com/api/' + '${AWS::Region}.${AWS::URLSuffix}/api/' ) } }, @@ -1122,6 +1172,14 @@ def test_managed_iam_role(self): {'PolicyName': 'DefaultRolePolicy', 'PolicyDocument': {'iam': 'policy'}} ] + # Verify the trust policy is specific to the region + assert cfn_role['Properties']['AssumeRolePolicyDocument'] == { + 'Statement': [{'Action': 'sts:AssumeRole', + 'Effect': 'Allow', + 'Principal': {'Service': 'lambda.amazonaws.com'}, + 'Sid': ''}], + 'Version': '2012-10-17'} + # Ensure the RoleName is not in the resource properties # so we don't require CAPABILITY_NAMED_IAM. assert 'RoleName' not in cfn_role['Properties'] @@ -1142,7 +1200,8 @@ def third(event, context): project_dir='.', autogen_policy=True, api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') roles = [resource for resource in template['Resources'].values() if resource['Type'] == 'AWS::IAM::Role'] assert len(roles) == 1 @@ -1167,7 +1226,8 @@ def test_vpc_config_added_to_function(self, sample_app_lambda_only): api_gateway_stage='api', security_group_ids=['sg1', 'sg2'], subnet_ids=['sn1', 'sn2']) - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') resources = template['Resources'].values() lambda_fns = [resource for resource in resources if resource['Type'] == 'AWS::Serverless::Function'] @@ -1185,8 +1245,9 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) with pytest.raises(NotImplementedError) as excinfo: - self.generate_template(config, 'dev') + self.generate_template(config, options, 'dev') # Should mention the decorator name. assert '@app.on_s3_event' in str(excinfo.value) # Should mention you can use `chalice deploy`. @@ -1200,7 +1261,8 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') sns_handler = template['Resources']['Handler'] assert sns_handler['Properties']['Events'] == { 'HandlerSnsSubscription': { @@ -1208,7 +1270,8 @@ def handler(event): 'Properties': { 'Topic': { 'Fn::Sub': ( - 'arn:aws:sns:${AWS::Region}:${AWS::AccountId}:foo' + 'arn:${AWS::Partition}:sns:${AWS::Region}' + ':${AWS::AccountId}:foo' ) } }, @@ -1225,7 +1288,8 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') sns_handler = template['Resources']['Handler'] assert sns_handler['Properties']['Events'] == { 'HandlerSnsSubscription': { @@ -1244,7 +1308,8 @@ def handler(event): config = Config.create(chalice_app=sample_app, project_dir='.', api_gateway_stage='api') - template = self.generate_template(config, 'dev') + options = PackageOptions(mock.Mock(spec=TypedAWSClient)) + template = self.generate_template(config, options, 'dev') sns_handler = template['Resources']['Handler'] assert sns_handler['Properties']['Events'] == { 'HandlerSqsEventSource': { @@ -1252,7 +1317,8 @@ def handler(event): 'Properties': { 'Queue': { 'Fn::Sub': ( - 'arn:aws:sqs:${AWS::Region}:${AWS::AccountId}:foo' + 'arn:${AWS::Partition}:sqs:${AWS::Region}' + ':${AWS::AccountId}:foo' ) }, 'BatchSize': 5, diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py index b0c3d54f15..a2b1e4a1ca 100644 --- a/tests/unit/test_policy.py +++ b/tests/unit/test_policy.py @@ -31,7 +31,7 @@ def test_app_policy_generator_vpc_policy(): 'logs:CreateLogStream', 'logs:PutLogEvents'], 'Effect': 'Allow', - 'Resource': 'arn:aws:logs:*:*:*'}, + 'Resource': 'arn:*:logs:*:*:*'}, {'Action': ['ec2:CreateNetworkInterface', 'ec2:DescribeNetworkInterfaces', 'ec2:DetachNetworkInterface', diff --git a/tests/unit/vendored/__init__.py b/tests/unit/vendored/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/vendored/botocore/__init__.py b/tests/unit/vendored/botocore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/vendored/botocore/test_regions.py b/tests/unit/vendored/botocore/test_regions.py new file mode 100644 index 0000000000..2b17347308 --- /dev/null +++ b/tests/unit/vendored/botocore/test_regions.py @@ -0,0 +1,253 @@ +# Copyright 2015 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +""" +This file was 'vendored' from botocore core +botocore/tests/unit/test_regions.py from commit +0c55d6c3f900fc856e818f06b31c22c6dbc56788. + +The vendoring/duplication was due to the concern of utilizing a unexposed class +internal to the botocore library for functionality necessary to implicitly +support partitions within the chalice microframework. More specifically the +determination of the dns suffix for service endpoints based on service and +region. +""" + +import pytest +from botocore.exceptions import NoRegionError + +from chalice.vendored.botocore import regions + + +@pytest.fixture +def endpoints_template(): + return { + 'partitions': [ + { + 'partition': 'aws', + 'dnsSuffix': 'amazonaws.com', + 'regionRegex': r'^(us|eu)\-\w+$', + 'defaults': { + 'hostname': '{service}.{region}.{dnsSuffix}' + }, + 'regions': { + 'us-foo': {'regionName': 'a'}, + 'us-bar': {'regionName': 'b'}, + 'eu-baz': {'regionName': 'd'} + }, + 'services': { + 'ec2': { + 'endpoints': { + 'us-foo': {}, + 'us-bar': {}, + 'eu-baz': {}, + 'd': {} + } + }, + 's3': { + 'defaults': { + 'sslCommonName': '{service}.{region}.{dnsSuffix}' + }, + 'endpoints': { + 'us-foo': { + 'sslCommonName': + '{region}.{service}.{dnsSuffix}' + }, + 'us-bar': {}, + 'eu-baz': {'hostname': 'foo'} + } + }, + 'not-regionalized': { + 'isRegionalized': False, + 'partitionEndpoint': 'aws', + 'endpoints': { + 'aws': {'hostname': 'not-regionalized'}, + 'us-foo': {}, + 'eu-baz': {} + } + }, + 'non-partition': { + 'partitionEndpoint': 'aws', + 'endpoints': { + 'aws': {'hostname': 'host'}, + 'us-foo': {} + } + }, + 'merge': { + 'defaults': { + 'signatureVersions': ['v2'], + 'protocols': ['http'] + }, + 'endpoints': { + 'us-foo': {'signatureVersions': ['v4']}, + 'us-bar': {'protocols': ['https']} + } + } + } + }, + { + 'partition': 'foo', + 'dnsSuffix': 'foo.com', + 'regionRegex': r'^(foo)\-\w+$', + 'defaults': { + 'hostname': '{service}.{region}.{dnsSuffix}', + 'protocols': ['http'], + 'foo': 'bar' + }, + 'regions': { + 'foo-1': {'regionName': '1'}, + 'foo-2': {'regionName': '2'}, + 'foo-3': {'regionName': '3'} + }, + 'services': { + 'ec2': { + 'endpoints': { + 'foo-1': { + 'foo': 'baz' + }, + 'foo-2': {}, + 'foo-3': {} + } + } + } + } + ] + } + + +def test_ensures_region_is_not_none(endpoints_template): + with pytest.raises(NoRegionError): + resolver = regions.EndpointResolver(endpoints_template) + resolver.construct_endpoint('foo', None) + + +def test_ensures_required_keys_present(endpoints_template): + with pytest.raises(ValueError): + regions.EndpointResolver({}) + + +def test_returns_empty_list_when_listing_for_different_partition( + endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + assert resolver.get_available_endpoints('ec2', 'bar') == [] + + +def test_returns_empty_list_when_no_service_found(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + assert resolver.get_available_endpoints('what?') == [] + + +def test_gets_endpoint_names(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.get_available_endpoints('ec2', allow_non_regional=True) + assert sorted(result) == ['d', 'eu-baz', 'us-bar', 'us-foo'] + + +def test_gets_endpoint_names_for_partition(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.get_available_endpoints( + 'ec2', allow_non_regional=True, partition_name='foo') + assert sorted(result) == ['foo-1', 'foo-2', 'foo-3'] + + +def test_list_regional_endpoints_only(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.get_available_endpoints( + 'ec2', allow_non_regional=False) + assert sorted(result) == ['eu-baz', 'us-bar', 'us-foo'] + + +def test_returns_none_when_no_match(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + assert resolver.construct_endpoint('foo', 'baz') is None + + +def test_constructs_regionalized_endpoints_for_exact_matches( + endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('not-regionalized', 'eu-baz') + assert result['hostname'] == 'not-regionalized.eu-baz.amazonaws.com' + assert result['partition'] == 'aws' + assert result['endpointName'] == 'eu-baz' + + +def test_constructs_partition_endpoints_for_real_partition_region( + endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('not-regionalized', 'us-bar') + assert result['hostname'] == 'not-regionalized' + assert result['partition'] == 'aws' + assert result['endpointName'] == 'aws' + + +def test_constructs_partition_endpoints_for_regex_match(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('not-regionalized', 'us-abc') + assert result['hostname'] == 'not-regionalized' + + +def test_constructs_endpoints_for_regionalized_regex_match(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('s3', 'us-abc') + assert result['hostname'] == 's3.us-abc.amazonaws.com' + + +def test_constructs_endpoints_for_unknown_service_but_known_region( + endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('unknown', 'us-foo') + assert result['hostname'] == 'unknown.us-foo.amazonaws.com' + + +def test_merges_service_keys(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + us_foo = resolver.construct_endpoint('merge', 'us-foo') + us_bar = resolver.construct_endpoint('merge', 'us-bar') + assert us_foo['protocols'] == ['http'] + assert us_foo['signatureVersions'] == ['v4'] + assert us_bar['protocols'] == ['https'] + assert us_bar['signatureVersions'] == ['v2'] + + +def test_merges_partition_default_keys_with_no_overwrite(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + resolved = resolver.construct_endpoint('ec2', 'foo-1') + assert resolved['foo'] == 'baz' + assert resolved['protocols'] == ['http'] + + +def test_merges_partition_default_keys_with_overwrite(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + resolved = resolver.construct_endpoint('ec2', 'foo-2') + assert resolved['foo'] == 'bar' + assert resolved['protocols'] == ['http'] + + +def test_gives_hostname_and_common_name_unaltered(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('s3', 'eu-baz') + assert result['sslCommonName'] == 's3.eu-baz.amazonaws.com' + assert result['hostname'] == 'foo' + + +def tests_uses_partition_endpoint_when_no_region_provided(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('not-regionalized') + assert result['hostname'] == 'not-regionalized' + assert result['endpointName'] == 'aws' + + +def test_returns_dns_suffix_if_available(endpoints_template): + resolver = regions.EndpointResolver(endpoints_template) + result = resolver.construct_endpoint('not-regionalized') + assert result['dnsSuffix'] == 'amazonaws.com'