Fix order of resource checks for CF deployment in single process (loc…

…alstack#2136)

localstack/services/apigateway/apigateway_listener.py

@@ -204,7 +204,7 @@ def invoke_rest_api(api_id, stage, method, invocation_path, data, headers, path=
                 new_request = aws_stack.render_velocity_template(template, data) + '&QueueName=%s' % queue
                 headers = aws_stack.mock_aws_request_headers(service='sqs', region_name=region_name)
 
-                url = urljoin(TEST_SQS_URL, '%s/%s' % (account_id, queue))
+                url = urljoin(TEST_SQS_URL, 'queue/%s' % queue)
                 result = common.make_http_request(url, method='POST', headers=headers, data=new_request)
                 return result
 

localstack/services/awslambda/lambda_api.py

@@ -58,7 +58,7 @@
 LAMBDA_DEFAULT_TIMEOUT = 3
 # default handler and runtime
 LAMBDA_DEFAULT_HANDLER = 'handler.handler'
-LAMBDA_DEFAULT_RUNTIME = LAMBDA_RUNTIME_PYTHON27
+LAMBDA_DEFAULT_RUNTIME = LAMBDA_RUNTIME_PYTHON36
 LAMBDA_DEFAULT_STARTING_POSITION = 'LATEST'
 LAMBDA_ZIP_FILE_NAME = 'original_lambda_archive.zip'
 LAMBDA_JAR_FILE_NAME = 'original_lambda_archive.jar'

localstack/services/cloudformation/cloudformation_starter.py

@@ -214,16 +214,12 @@ def _parse_and_create_resource(logical_id, resource_json, resources_map, region_
         if resource and not update and not force_create:
             return resource
 
-        # check whether this resource needs to be deployed
-        resource_map_new = dict(resources_map._resource_json_map)
-        resource_map_new[logical_id] = resource_json
-        should_be_created = template_deployer.should_be_deployed(logical_id, resource_map_new, stack_name)
-
         # fix resource ARNs, make sure to convert account IDs 000000000000 to 123456789012
         resource_json_arns_fixed = clone(json_safe(convert_objs_to_ids(resource_json)))
         set_moto_account_ids(resource_json_arns_fixed)
 
         # create resource definition and store CloudFormation metadata in moto
+        moto_create_error = None
         if (resource or update) and not force_create:
             parse_and_update_resource_orig(logical_id,
                 resource_json_arns_fixed, resources_map, region_name)
@@ -233,11 +229,20 @@ def _parse_and_create_resource(logical_id, resource_json, resources_map, region_
                     resource_json_arns_fixed, resources_map, region_name)
                 resource.logical_id = logical_id
             except Exception as e:
-                if should_be_created:
-                    raise
-                else:
-                    LOG.info('Error on moto CF resource creation. Ignoring, as should_be_created=%s: %s' %
-                             (should_be_created, e))
+                moto_create_error = e
+
+        # check whether this resource needs to be deployed
+        resource_map_new = dict(resources_map._resource_json_map)
+        resource_map_new[logical_id] = resource_json
+        should_be_created = template_deployer.should_be_deployed(logical_id, resource_map_new, stack_name)
+
+        # check for moto creation errors and raise an exception if needed
+        if moto_create_error:
+            if should_be_created:
+                raise moto_create_error
+            else:
+                LOG.info('Error on moto CF resource creation. Ignoring, as should_be_created=%s: %s' %
+                         (should_be_created, moto_create_error))
 
         # Fix for moto which sometimes hard-codes region name as 'us-east-1'
         if hasattr(resource, 'region_name') and resource.region_name != region_name:

localstack/services/s3/s3_starter.py

@@ -73,6 +73,19 @@ def s3_update_acls(self, request, query, bucket_name, key_name):
         if acl:
             key.set_acl(acl)
 
+    # patch Bucket.create_from_cloudformation_json in moto
+
+    @classmethod
+    def Bucket_create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
+        result = create_from_cloudformation_json_orig(resource_name, cloudformation_json, region_name)
+        tags = cloudformation_json['Properties'].get('Tags', [])
+        for tag in tags:
+            result.tags.tag_set.tags.append(s3_models.FakeTag(tag['Key'], tag['Value']))
+        return result
+
+    create_from_cloudformation_json_orig = s3_models.FakeBucket.create_from_cloudformation_json
+    s3_models.FakeBucket.create_from_cloudformation_json = Bucket_create_from_cloudformation_json
+
     # patch _key_response_post(..)
 
     def s3_key_response_post(self, request, body, bucket_name, query, key_name, *args, **kwargs):

localstack/utils/aws/aws_stack.py

@@ -373,6 +373,11 @@ def get_iam_role(resource, env=None):
     return 'role-%s' % resource
 
 
+def secretsmanager_secret_arn(secret_name, account_id=None, region_name=None):
+    pattern = 'arn:aws:secretsmanager:%s:%s:secret:%s'
+    return _resource_arn(secret_name, pattern, account_id=account_id, region_name=region_name)
+
+
 def cloudformation_stack_arn(stack_name, account_id=None, region_name=None):
     pattern = 'arn:aws:cloudformation:%s:%s:stack/%s/id-1234'
     return _resource_arn(stack_name, pattern, account_id=account_id, region_name=region_name)

localstack/utils/testutil.py

@@ -14,7 +14,6 @@
 from localstack.services.awslambda.lambda_api import (get_handler_file_from_name, LAMBDA_DEFAULT_HANDLER,
     LAMBDA_DEFAULT_RUNTIME, LAMBDA_DEFAULT_STARTING_POSITION, LAMBDA_DEFAULT_TIMEOUT)
 
-
 ARCHIVE_DIR_PREFIX = 'lambda.archive.'
 
 

requirements.txt

@@ -7,7 +7,7 @@
 # (3) The remaining dependencies (those without markers) are only included in
 # the "full" version of the pip package (i.e., "pip install localstack[full]").
 
-airspeed==0.5.10
+airspeed>=0.5.14
 # Use our "ext" version until this bug is fixed: https://github.com/awslabs/amazon-kinesis-client-python/issues/99
 amazon_kclpy-ext==1.5.1
 # amazon-kclpy==1.5.1

tests/integration/test_api_gateway.py

@@ -127,8 +127,7 @@ def test_api_gateway_sqs_integration_with_event_source(self):
 
         # create API Gateway and connect it to the target queue
         result = connect_api_gateway_to_sqs(
-            'test_gateway4',
-            stage_name=self.TEST_STAGE_NAME,
+            'test_gateway4', stage_name=self.TEST_STAGE_NAME,
             queue_arn=self.TEST_SQS_QUEUE, path=self.API_PATH_DATA_INBOUND)
 
         # create event source for sqs lambda processor
@@ -152,11 +151,11 @@ def test_api_gateway_sqs_integration_with_event_source(self):
         parsed_json = xmltodict.parse(result.content)
         result = parsed_json['SendMessageResponse']['SendMessageResult']
 
-        attr_md5 = result['MD5OfMessageAttributes']
         body_md5 = result['MD5OfMessageBody']
+        attr_md5 = result['MD5OfMessageAttributes']
 
-        self.assertEqual(attr_md5, 'd41d8cd98f00b204e9800998ecf8427e')
         self.assertEqual(body_md5, 'b639f52308afd65866c86f274c59033f')
+        self.assertEqual(attr_md5, 'd41d8cd98f00b204e9800998ecf8427e')
 
     def test_api_gateway_sqs_integration(self):
         # create target SQS stream