Merge branch 'f-partitions'

PR #1417.

* f-partitions:
  Add changelog entry for non aws partitions support
  Fixup failing tests and formatting
  Implicit support for partitions as per #792

.changes/next-release/43213435694-enhancement-regions-86601.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "regions",
+  "description": "Add support for non `aws` partitions including aws-cn and aws-us-gov (#792)."
+}

.pylintrc

@@ -9,7 +9,7 @@
 
 # Add files or directories to the blacklist. They should be base names, not
 # paths.
-ignore=compat.py
+ignore=compat.py,regions.py
 
 # Pickle collected data for later comparisons.
 persistent=yes

Makefile

@@ -6,14 +6,14 @@ TESTS=tests/unit tests/functional tests/integration
 check:
 	###### FLAKE8 #####
 	# No unused imports, no undefined vars,
-	flake8 --ignore=E731,W503,W504 --exclude chalice/__init__.py,chalice/compat.py --max-complexity 10 chalice/
+	flake8 --ignore=E731,W503,W504 --exclude chalice/__init__.py,chalice/compat.py,chalice/vendored/botocore/regions.py --max-complexity 10 chalice/
 	flake8 --ignore=E731,W503,W504,F401 --max-complexity 10 chalice/compat.py
 	flake8 tests/unit/ tests/functional/ tests/integration tests/aws
 	#
 	# Proper docstring conventions according to pep257
 	#
 	#
-	pydocstyle --add-ignore=D100,D101,D102,D103,D104,D105,D204,D301 chalice/
+	pydocstyle --add-ignore=D100,D101,D102,D103,D104,D105,D204,D301 --match='(?!(test_|regions)).*\.py' chalice/
 
 pylint:
 	###### PYLINT ######

chalice/cli/__init__.py

@@ -530,14 +530,19 @@ def generate_models(ctx, stage):
               type=click.Choice(['json', 'yaml'], case_sensitive=False),
               help=('Specify if the generated template should be serialized '
                     'as either JSON or YAML.  CloudFormation only.'))
+@click.option('--profile', help='Override profile at packaging time.')
 @click.argument('out')
 @click.pass_context
 def package(ctx, single_file, stage, merge_template,
-            out, pkg_format, template_format):
-    # type: (click.Context, bool, str, str, str, str, str) -> None
+            out, pkg_format, template_format, profile):
+    # type: (click.Context, bool, str, str, str, str, str, str) -> None
     factory = ctx.obj['factory']  # type: CLIFactory
+    factory.profile = profile
     config = factory.create_config_obj(stage)
-    packager = factory.create_app_packager(config, pkg_format, template_format,
+    options = factory.create_package_options()
+    packager = factory.create_app_packager(config, options,
+                                           pkg_format,
+                                           template_format,
                                            merge_template)
     if pkg_format == 'terraform' and (merge_template or
                                       single_file or

chalice/cli/factory.py

@@ -17,6 +17,7 @@
 from chalice.config import DeployedResources  # noqa
 from chalice.package import create_app_packager
 from chalice.package import AppPackager  # noqa
+from chalice.package import PackageOptions
 from chalice.constants import DEFAULT_STAGE_NAME
 from chalice.constants import DEFAULT_APIGATEWAY_STAGE_NAME
 from chalice.constants import DEFAULT_ENDPOINT_TYPE
@@ -75,6 +76,7 @@ def _inject_large_request_body_filter():
 
 class NoSuchFunctionError(Exception):
     """The specified function could not be found."""
+
     def __init__(self, name):
         # type: (str) -> None
         self.name = name
@@ -185,11 +187,11 @@ def _validate_config_from_disk(self, config):
         except ValueError:
             raise UnknownConfigFileVersion(string_version)
 
-    def create_app_packager(self, config, package_format, template_format,
-                            merge_template=None):
-        # type: (Config, str, str, OptStr) -> AppPackager
+    def create_app_packager(self, config, options, package_format,
+                            template_format, merge_template=None):
+        # type: (Config, PackageOptions, str, str, OptStr) -> AppPackager
         return create_app_packager(
-            config, package_format, template_format,
+            config, options, package_format, template_format,
             merge_template=merge_template)
 
     def create_log_retriever(self, session, lambda_arn, follow_logs):
@@ -298,3 +300,10 @@ def load_project_config(self):
     def create_local_server(self, app_obj, config, host, port):
         # type: (Chalice, Config, str, int) -> local.LocalDevServer
         return local.create_local_server(app_obj, config, host, port)
+
+    def create_package_options(self):
+        # type: () -> PackageOptions
+        """Create the package options that are required to target regions."""
+        s = Session(profile=self.profile)
+        client = TypedAWSClient(session=s)
+        return PackageOptions(client)

chalice/constants.py

@@ -79,7 +79,7 @@ def index():
         "logs:CreateLogStream",
         "logs:PutLogEvents"
     ],
-    "Resource": "arn:aws:logs:*:*:*"
+    "Resource": "arn:*:logs:*:*:*"
 }
 
 
@@ -114,7 +114,7 @@ def index():
                 "s3:GetObjectVersion",
                 "s3:PutObject"
             ],
-            "Resource": "arn:aws:s3:::*",
+            "Resource": "arn:*:s3:::*",
             "Effect": "Allow"
         }
     ]
@@ -248,5 +248,5 @@ def index():
     "Action": [
         "execute-api:ManageConnections"
     ],
-    "Resource": "arn:aws:execute-api:*:*:*/@connections/*"
+    "Resource": "arn:*:execute-api:*:*:*/@connections/*"
 }

chalice/deploy/appgraph.py

@@ -182,7 +182,7 @@ def _get_default_private_api_policy(self, config):
             "Effect": "Allow",
             "Principal": "*",
             "Action": "execute-api:Invoke",
-            "Resource": "arn:aws:execute-api:*:*:*",
+            "Resource": "arn:*:execute-api:*:*:*",
             "Condition": {
                 "StringEquals": {
                     "aws:SourceVpce": config.api_gateway_endpoint_vpce

chalice/deploy/executor.py

@@ -123,9 +123,34 @@ def _do_builtinfunction(self, instruction):
             value = resolved_args[0]
             parts = value.split(':')
             result = {
+                'partition': parts[1],
                 'service': parts[2],
                 'region': parts[3],
                 'account_id': parts[4],
+                'dns_suffix': self._client.endpoint_dns_suffix(parts[2],
+                                                               parts[3])
+            }
+            self.variables[instruction.output_var] = result
+        elif instruction.function_name == 'interrogate_profile':
+            region = self._client.region_name
+            result = {
+                'partition': self._client.partition_name,
+                'region': region,
+                'dns_suffix': self._client.endpoint_dns_suffix('apigateway',
+                                                               region)
+            }
+            self.variables[instruction.output_var] = result
+        elif instruction.function_name == 'service_principal':
+            resolved_args = self._variable_resolver.resolve_variables(
+                instruction.args, self.variables)
+            service_name = resolved_args[0]
+            region_name = self._client.region_name
+            dns_suffix = self._client.endpoint_dns_suffix(service_name,
+                                                          region_name)
+            result = {
+                'principal': self._client.service_principal(service_name,
+                                                            region_name,
+                                                            dns_suffix)
             }
             self.variables[instruction.output_var] = result
         else:
@@ -196,7 +221,6 @@ def _resolve_list(self, value, variables):
 # The dev commands don't have any backwards compatibility guarantees
 # so we can alter this output as needed.
 class DisplayOnlyExecutor(BaseExecutor):
-
     # Max length of bytes object before we truncate with '<bytes>'
     _MAX_BYTE_LENGTH = 30
     _LINE_VERTICAL = u'\u2502'

chalice/deploy/planner.py

@@ -1,4 +1,5 @@
 # pylint: disable=too-many-lines
+import re
 import json
 from collections import OrderedDict
 
@@ -454,10 +455,37 @@ def _plan_managediamrole(self, resource):
         varname = '%s_role_arn' % resource.role_name
         if not role_exists:
             return [
+                models.BuiltinFunction(
+                    'service_principal',
+                    ['lambda'],
+                    output_var='lambda_service_principal',
+                ),
+                models.JPSearch('principal',
+                                input_var='lambda_service_principal',
+                                output_var='lambda_principal'),
+                models.StoreValue(
+                    name='lambda_principal',
+                    value=StringFormat('{lambda_principal}',
+                                       ['lambda_principal']),
+                ),
+                models.StoreValue(
+                    name='lambda_trust_policy',
+                    value={
+                        "Version": "2012-10-17",
+                        "Statement": [{
+                            "Sid": "",
+                            "Effect": "Allow",
+                            "Principal": {
+                                "Service": Variable('lambda_principal')
+                            },
+                            "Action": "sts:AssumeRole"
+                        }]
+                    },
+                ),
                 (models.APICall(
                     method_name='create_role',
                     params={'name': resource.role_name,
-                            'trust_policy': resource.trust_policy,
+                            'trust_policy': Variable('lambda_trust_policy'),
                             'policy': document},
                     output_var=varname,
                 ), "Creating IAM role: %s\n" % resource.role_name),
@@ -507,7 +535,7 @@ def _plan_snslambdasubscription(self, resource):
         subscribe_varname = '%s_subscription_arn' % resource.resource_name
 
         instruction_for_topic_arn = []  # type: List[InstructionMsg]
-        if resource.topic.startswith('arn:aws:sns:'):
+        if re.match(r"^arn:aws[a-z\-]*:sns:", resource.topic):
             instruction_for_topic_arn += [
                 models.StoreValue(
                     name=topic_arn_varname,
@@ -530,13 +558,16 @@ def _plan_snslambdasubscription(self, resource):
                 models.JPSearch('region',
                                 input_var='parsed_lambda_arn',
                                 output_var='region_name'),
+                models.JPSearch('partition',
+                                input_var='parsed_lambda_arn',
+                                output_var='partition'),
                 models.StoreValue(
                     name=topic_arn_varname,
                     value=StringFormat(
-                        'arn:aws:sns:{region_name}:{account_id}:%s' % (
+                        'arn:{partition}:sns:{region_name}:{account_id}:%s' % (
                             resource.topic
                         ),
-                        ['region_name', 'account_id'],
+                        ['partition', 'region_name', 'account_id'],
                     ),
                 ),
             ]
@@ -633,13 +664,16 @@ def _plan_sqseventsource(self, resource):
             models.JPSearch('region',
                             input_var='parsed_lambda_arn',
                             output_var='region_name'),
+            models.JPSearch('partition',
+                            input_var='parsed_lambda_arn',
+                            output_var='partition'),
             models.StoreValue(
                 name=queue_arn_varname,
                 value=StringFormat(
-                    'arn:aws:sqs:{region_name}:{account_id}:%s' % (
+                    'arn:{partition}:sqs:{region_name}:{account_id}:%s' % (
                         resource.queue
                     ),
-                    ['region_name', 'account_id'],
+                    ['partition', 'region_name', 'account_id'],
                 ),
             ),
         ]  # type: List[InstructionMsg]
@@ -834,11 +868,11 @@ def _inject_websocket_integrations(self, configs):
                 models.StoreValue(
                     name='websocket-%s-integration-lambda-path' % key,
                     value=StringFormat(
-                        'arn:aws:apigateway:{region_name}:lambda:path/'
-                        '2015-03-31/functions/arn:aws:lambda:{region_name}:'
-                        '{account_id}:function:%s/'
-                        'invocations' % config['name'],
-                        ['region_name', 'account_id'],
+                        'arn:{partition}:apigateway:{region_name}:lambda:path/'
+                        '2015-03-31/functions/arn:{partition}'
+                        ':lambda:{region_name}:{account_id}:function'
+                        ':%s/invocations' % config['name'],
+                        ['partition', 'region_name', 'account_id'],
                     ),
                 ),
             )
@@ -895,6 +929,12 @@ def _plan_websocketapi(self, resource):
             models.JPSearch('region',
                             input_var='parsed_lambda_arn',
                             output_var='region_name'),
+            models.JPSearch('partition',
+                            input_var='parsed_lambda_arn',
+                            output_var='partition'),
+            models.JPSearch('dns_suffix',
+                            input_var='parsed_lambda_arn',
+                            output_var='dns_suffix'),
         ]  # type: List[InstructionMsg]
 
         # There's also a set of instructions that are needed
@@ -905,8 +945,8 @@ def _plan_websocketapi(self, resource):
                 name='websocket_api_url',
                 value=StringFormat(
                     'wss://{websocket_api_id}.execute-api.{region_name}'
-                    '.amazonaws.com/%s/' % resource.api_gateway_stage,
-                    ['websocket_api_id', 'region_name'],
+                    '.{dns_suffix}/%s/' % resource.api_gateway_stage,
+                    ['websocket_api_id', 'region_name', 'dns_suffix'],
                 ),
             ),
             models.RecordResourceVariable(
@@ -1047,6 +1087,12 @@ def _plan_restapi(self, resource):
             models.JPSearch('region',
                             input_var='parsed_lambda_arn',
                             output_var='region_name'),
+            models.JPSearch('partition',
+                            input_var='parsed_lambda_arn',
+                            output_var='partition'),
+            models.JPSearch('dns_suffix',
+                            input_var='parsed_lambda_arn',
+                            output_var='dns_suffix'),
             # The swagger doc uses the 'api_handler_lambda_arn'
             # var name so we need to make sure we populate this variable
             # before importing the rest API.
@@ -1086,8 +1132,8 @@ def _plan_restapi(self, resource):
                 name='rest_api_url',
                 value=StringFormat(
                     'https://{rest_api_id}.execute-api.{region_name}'
-                    '.amazonaws.com/%s/' % resource.api_gateway_stage,
-                    ['rest_api_id', 'region_name'],
+                    '.{dns_suffix}/%s/' % resource.api_gateway_stage,
+                    ['rest_api_id', 'region_name', 'dns_suffix'],
                 ),
             ),
             models.RecordResourceVariable(

chalice/deploy/swagger.py

@@ -172,9 +172,11 @@ def _uri(self, lambda_arn=None):
         # type: (Optional[str]) -> Any
         if lambda_arn is None:
             lambda_arn = self._deployed_resources['api_handler_arn']
-        return ('arn:aws:apigateway:{region}:lambda:path/2015-03-31'
+        partition = lambda_arn.split(':')[1]
+        return ('arn:{partition}:apigateway:{region}:lambda:path/2015-03-31'
                 '/functions/{lambda_arn}/invocations').format(
-                    region=self._region, lambda_arn=lambda_arn)
+                    partition=partition, region=self._region,
+                    lambda_arn=lambda_arn)
 
     def _generate_apig_integ(self, view):
         # type: (RouteEntry) -> Dict[str, Any]
@@ -250,7 +252,8 @@ def _uri(self, lambda_arn=None):
         # type: (Optional[str]) -> Any
         return {
             'Fn::Sub': (
-                'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31'
+                'arn:${AWS::Partition}:apigateway:${AWS::Region}'
+                ':lambda:path/2015-03-31'
                 '/functions/${APIHandler.Arn}/invocations'
             )
         }
@@ -259,7 +262,8 @@ def _auth_uri(self, authorizer):
         # type: (ChaliceAuthorizer) -> Any
         return {
             'Fn::Sub': (
-                'arn:aws:apigateway:${AWS::Region}:lambda:path/2015-03-31'
+                'arn:${AWS::Partition}:apigateway:${AWS::Region}'
+                ':lambda:path/2015-03-31'
                 '/functions/${%s.Arn}/invocations' % to_cfn_resource_name(
                     authorizer.name)
             )
@@ -274,18 +278,18 @@ def __init__(self):
     def _uri(self, lambda_arn=None):
         # type: (Optional[str]) -> Any
         return StringFormat(
-            'arn:aws:apigateway:{region_name}:lambda:path/2015-03-31'
+            'arn:{partition}:apigateway:{region_name}:lambda:path/2015-03-31'
             '/functions/{api_handler_lambda_arn}/invocations',
-            ['region_name', 'api_handler_lambda_arn'],
+            ['partition', 'region_name', 'api_handler_lambda_arn'],
         )
 
     def _auth_uri(self, authorizer):
         # type: (ChaliceAuthorizer) -> Any
         varname = '%s_lambda_arn' % authorizer.name
         return StringFormat(
-            'arn:aws:apigateway:{region_name}:lambda:path/2015-03-31'
+            'arn:{partition}:apigateway:{region_name}:lambda:path/2015-03-31'
             '/functions/{%s}/invocations' % varname,
-            ['region_name', varname],
+            ['partition', 'region_name', varname],
         )