From 8062e146412f46ec9a6622121204a8882b7febfc Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 23 Nov 2021 14:18:36 -0800 Subject: [PATCH 01/11] Consume image asset metadata --- .../samlib/resource_metadata_normalizer.py | 70 ++++++++++++++++++- .../test_resource_metadata_normalizer.py | 23 ++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 0505103924..aceea17c0a 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -3,13 +3,23 @@ """ import logging +import os RESOURCES_KEY = "Resources" PROPERTIES_KEY = "Properties" METADATA_KEY = "Metadata" + ASSET_PATH_METADATA_KEY = "aws:asset:path" ASSET_PROPERTY_METADATA_KEY = "aws:asset:property" +IMAGE_ASSET_PROPERTY = "Code.ImageUri" +ASSET_DOCKERFILE_PATH_KEY = "aws:asset:dockerfile-path" +ASSET_DOCKERFILE_BUILD_ARGS_KEY = "aws:asset:docker-build-args" + +SAM_METADATA_DOCKERFILE_KEY = "Dockerfile" +SAM_METADATA_DOCKER_CONTEXT_KEY = "DockerContext" +SAM_METADATA_DOCKER_BUILD_ARGS_KEY = "DockerBuildArgs" + LOG = logging.getLogger(__name__) @@ -31,9 +41,15 @@ def normalize(template_dict): for logical_id, resource in resources.items(): resource_metadata = resource.get(METADATA_KEY, {}) - asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY) + if asset_property == IMAGE_ASSET_PROPERTY: + asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata) + ResourceMetadataNormalizer._update_resource_image_asset_metadata(resource_metadata, asset_metadata) + asset_path = logical_id.lower() + else: + asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) + ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id) @staticmethod @@ -56,10 +72,60 @@ def _replace_property(property_key, property_value, resource, logical_id): """ if property_key and property_value: - resource.get(PROPERTIES_KEY, {})[property_key] = property_value + nested_keys = property_key.split(".") + target_dict = resource.get(PROPERTIES_KEY, {}) + while len(nested_keys) > 1: + key = nested_keys.pop(0) + target_dict[key] = {} + target_dict = target_dict[key] + target_dict[nested_keys[0]] = property_value elif property_key or property_value: LOG.info( "WARNING: Ignoring Metadata for Resource %s. Metadata contains only aws:asset:path or " "aws:assert:property but not both", logical_id, ) + + @staticmethod + def _extract_image_asset_metadata(metadata): + """ + Extract/create relevant metadata properties for image assets + + Parameters + ---------- + metadata dict + Metadata to use for extracting image assets properties + + Returns + ------- + dict + metadata properties for image-type lambda function + + """ + path_from_asset, dockerfile = os.path.split(metadata.get(ASSET_DOCKERFILE_PATH_KEY)) + stripped_path = path_from_asset.lstrip(os.sep) + asset_path = metadata.get(ASSET_PATH_METADATA_KEY) + dockerfile_context = os.path.join(asset_path, stripped_path) + return { + SAM_METADATA_DOCKERFILE_KEY: dockerfile, + SAM_METADATA_DOCKER_CONTEXT_KEY: dockerfile_context, + SAM_METADATA_DOCKER_BUILD_ARGS_KEY: metadata.get(ASSET_DOCKERFILE_BUILD_ARGS_KEY, {}), + } + + @staticmethod + def _update_resource_image_asset_metadata(metadata, updated_values): + """ + Update the metadata values for image-type lambda functions + + This method will mutate the template + + Parameters + ---------- + metadata dict + Metadata dict to be updated + updated_values dict + Dict of key-value pairs to append to the existing metadata + + """ + for key, val in updated_values.items(): + metadata[key] = val diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index e87d9d31d7..1770154a44 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -37,6 +37,29 @@ def test_replace_all_resources_that_contain_metadata(self): self.assertEqual("new path", template_data["Resources"]["Function1"]["Properties"]["Code"]) self.assertEqual("super cool path", template_data["Resources"]["Resource2"]["Properties"]["SomeRandomProperty"]) + def test_replace_all_resources_that_contain_image_metadata(self): + docker_build_args = {"arg1": "val1", "arg2": "val2"} + template_data = { + "Resources": { + "Function1": { + "Properties": {"Code": "some value"}, + "Metadata": { + "aws:asset:path": "/path/to/asset", + "aws:asset:property": "Code.ImageUri", + "aws:asset:dockerfile-path": "/path/to/Dockerfile", + "aws:asset:docker-build-args": docker_build_args, + }, + }, + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("function1", template_data["Resources"]["Function1"]["Properties"]["Code"]["ImageUri"]) + self.assertEqual("/path/to/asset/path/to", template_data["Resources"]["Function1"]["Metadata"]["DockerContext"]) + self.assertEqual("Dockerfile", template_data["Resources"]["Function1"]["Metadata"]["Dockerfile"]) + self.assertEqual(docker_build_args, template_data["Resources"]["Function1"]["Metadata"]["DockerBuildArgs"]) + def test_tempate_without_metadata(self): template_data = {"Resources": {"Function1": {"Properties": {"Code": "some value"}}}} From 1825a1f7aca656a19ef55654718d531a6e10f002 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 23 Nov 2021 16:27:25 -0800 Subject: [PATCH 02/11] Make tests use correct os seperator --- .../MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README | 1 + .../lib/samlib/test_resource_metadata_normalizer.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README diff --git a/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README b/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README new file mode 100644 index 0000000000..f6f5ac40c6 --- /dev/null +++ b/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README @@ -0,0 +1 @@ +This layer contains dependencies of function MyFunction and automatically added by AWS SAM CLI command 'sam sync' \ No newline at end of file diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index 1770154a44..1a6350d71b 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -1,4 +1,5 @@ from unittest import TestCase +import os from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer @@ -39,14 +40,16 @@ def test_replace_all_resources_that_contain_metadata(self): def test_replace_all_resources_that_contain_image_metadata(self): docker_build_args = {"arg1": "val1", "arg2": "val2"} + asset_path = os.path.join("path", "to", "asset") + dockerfile_path = os.path.join("path", "to", "Dockerfile") template_data = { "Resources": { "Function1": { "Properties": {"Code": "some value"}, "Metadata": { - "aws:asset:path": "/path/to/asset", + "aws:asset:path": asset_path, "aws:asset:property": "Code.ImageUri", - "aws:asset:dockerfile-path": "/path/to/Dockerfile", + "aws:asset:dockerfile-path": dockerfile_path, "aws:asset:docker-build-args": docker_build_args, }, }, @@ -55,8 +58,11 @@ def test_replace_all_resources_that_contain_image_metadata(self): ResourceMetadataNormalizer.normalize(template_data) + expected_docker_context_path = os.path.join("path", "to", "asset", "path", "to") self.assertEqual("function1", template_data["Resources"]["Function1"]["Properties"]["Code"]["ImageUri"]) - self.assertEqual("/path/to/asset/path/to", template_data["Resources"]["Function1"]["Metadata"]["DockerContext"]) + self.assertEqual( + expected_docker_context_path, template_data["Resources"]["Function1"]["Metadata"]["DockerContext"] + ) self.assertEqual("Dockerfile", template_data["Resources"]["Function1"]["Metadata"]["Dockerfile"]) self.assertEqual(docker_build_args, template_data["Resources"]["Function1"]["Metadata"]["DockerBuildArgs"]) From 66a854b9c40a2b32e260d31bf6551bbf7a909625 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 23 Nov 2021 16:37:25 -0800 Subject: [PATCH 03/11] Remove unnecessary code --- samcli/lib/samlib/resource_metadata_normalizer.py | 3 +-- tests/unit/lib/samlib/test_resource_metadata_normalizer.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index aceea17c0a..59875ff148 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -103,9 +103,8 @@ def _extract_image_asset_metadata(metadata): """ path_from_asset, dockerfile = os.path.split(metadata.get(ASSET_DOCKERFILE_PATH_KEY)) - stripped_path = path_from_asset.lstrip(os.sep) asset_path = metadata.get(ASSET_PATH_METADATA_KEY) - dockerfile_context = os.path.join(asset_path, stripped_path) + dockerfile_context = os.path.join(asset_path, path_from_asset) return { SAM_METADATA_DOCKERFILE_KEY: dockerfile, SAM_METADATA_DOCKER_CONTEXT_KEY: dockerfile_context, diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index 1a6350d71b..e4d8570e52 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -40,7 +40,7 @@ def test_replace_all_resources_that_contain_metadata(self): def test_replace_all_resources_that_contain_image_metadata(self): docker_build_args = {"arg1": "val1", "arg2": "val2"} - asset_path = os.path.join("path", "to", "asset") + asset_path = os.path.join("/path", "to", "asset") dockerfile_path = os.path.join("path", "to", "Dockerfile") template_data = { "Resources": { @@ -58,7 +58,7 @@ def test_replace_all_resources_that_contain_image_metadata(self): ResourceMetadataNormalizer.normalize(template_data) - expected_docker_context_path = os.path.join("path", "to", "asset", "path", "to") + expected_docker_context_path = os.path.join("/path", "to", "asset", "path", "to") self.assertEqual("function1", template_data["Resources"]["Function1"]["Properties"]["Code"]["ImageUri"]) self.assertEqual( expected_docker_context_path, template_data["Resources"]["Function1"]["Metadata"]["DockerContext"] From 3394e2ca919cdec70699ff0d80a37699b1c21674 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 23 Nov 2021 16:40:33 -0800 Subject: [PATCH 04/11] Remove build dir --- build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README | 1 - 1 file changed, 1 deletion(-) delete mode 100644 build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README diff --git a/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README b/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README deleted file mode 100644 index f6f5ac40c6..0000000000 --- a/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README +++ /dev/null @@ -1 +0,0 @@ -This layer contains dependencies of function MyFunction and automatically added by AWS SAM CLI command 'sam sync' \ No newline at end of file From 455ba860e4152db68e39abb94e4ae66dcf1f4cce Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 24 Nov 2021 17:51:43 -0800 Subject: [PATCH 05/11] Use pathlib and add test for windows paths --- .../samlib/resource_metadata_normalizer.py | 9 ++-- .../test_resource_metadata_normalizer.py | 46 +++++++++++++++++-- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 59875ff148..2b146f381f 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -3,7 +3,7 @@ """ import logging -import os +import pathlib RESOURCES_KEY = "Resources" PROPERTIES_KEY = "Properties" @@ -102,9 +102,10 @@ def _extract_image_asset_metadata(metadata): metadata properties for image-type lambda function """ - path_from_asset, dockerfile = os.path.split(metadata.get(ASSET_DOCKERFILE_PATH_KEY)) - asset_path = metadata.get(ASSET_PATH_METADATA_KEY) - dockerfile_context = os.path.join(asset_path, path_from_asset) + asset_path = pathlib.Path(metadata.get(ASSET_PATH_METADATA_KEY, "")) + dockerfile_path = pathlib.Path(metadata.get(ASSET_DOCKERFILE_PATH_KEY), "") + dockerfile, path_from_asset = dockerfile_path.stem, dockerfile_path.parent + dockerfile_context = str(pathlib.Path(asset_path.joinpath(path_from_asset))) return { SAM_METADATA_DOCKERFILE_KEY: dockerfile, SAM_METADATA_DOCKER_CONTEXT_KEY: dockerfile_context, diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index e4d8570e52..2ae968a7ec 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -1,5 +1,5 @@ +import pathlib from unittest import TestCase -import os from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer @@ -40,12 +40,48 @@ def test_replace_all_resources_that_contain_metadata(self): def test_replace_all_resources_that_contain_image_metadata(self): docker_build_args = {"arg1": "val1", "arg2": "val2"} - asset_path = os.path.join("/path", "to", "asset") - dockerfile_path = os.path.join("path", "to", "Dockerfile") + asset_path = pathlib.Path("/path", "to", "asset") + dockerfile_path = pathlib.Path("path", "to", "Dockerfile") template_data = { "Resources": { "Function1": { - "Properties": {"Code": "some value"}, + "Properties": { + "Code": { + "ImageUri": "Some Value", + } + }, + "Metadata": { + "aws:asset:path": asset_path, + "aws:asset:property": "Code.ImageUri", + "aws:asset:dockerfile-path": dockerfile_path, + "aws:asset:docker-build-args": docker_build_args, + }, + }, + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + expected_docker_context_path = str(pathlib.Path("/path", "to", "asset", "path", "to")) + self.assertEqual("function1", template_data["Resources"]["Function1"]["Properties"]["Code"]["ImageUri"]) + self.assertEqual( + expected_docker_context_path, template_data["Resources"]["Function1"]["Metadata"]["DockerContext"] + ) + self.assertEqual("Dockerfile", template_data["Resources"]["Function1"]["Metadata"]["Dockerfile"]) + self.assertEqual(docker_build_args, template_data["Resources"]["Function1"]["Metadata"]["DockerBuildArgs"]) + + def test_replace_all_resources_that_contain_image_metadata_windows_paths(self): + docker_build_args = {"arg1": "val1", "arg2": "val2"} + asset_path = "C:\\path\\to\\asset" + dockerfile_path = "rel/path/to/Dockerfile" + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": { + "ImageUri": "Some Value", + } + }, "Metadata": { "aws:asset:path": asset_path, "aws:asset:property": "Code.ImageUri", @@ -58,7 +94,7 @@ def test_replace_all_resources_that_contain_image_metadata(self): ResourceMetadataNormalizer.normalize(template_data) - expected_docker_context_path = os.path.join("/path", "to", "asset", "path", "to") + expected_docker_context_path = "C:\\path\\to\\asset/rel/path/to" self.assertEqual("function1", template_data["Resources"]["Function1"]["Properties"]["Code"]["ImageUri"]) self.assertEqual( expected_docker_context_path, template_data["Resources"]["Function1"]["Metadata"]["DockerContext"] From f23727596ad30a2c8bc85ff2c751a0123dec6916 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 24 Nov 2021 19:02:19 -0800 Subject: [PATCH 06/11] Update test --- tests/unit/lib/samlib/test_resource_metadata_normalizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py index 2ae968a7ec..c03f42ccdb 100644 --- a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -94,7 +94,7 @@ def test_replace_all_resources_that_contain_image_metadata_windows_paths(self): ResourceMetadataNormalizer.normalize(template_data) - expected_docker_context_path = "C:\\path\\to\\asset/rel/path/to" + expected_docker_context_path = str(pathlib.Path("C:\\path\\to\\asset").joinpath(pathlib.Path("rel/path/to"))) self.assertEqual("function1", template_data["Resources"]["Function1"]["Properties"]["Code"]["ImageUri"]) self.assertEqual( expected_docker_context_path, template_data["Resources"]["Function1"]["Metadata"]["DockerContext"] From 172a52edf49a38e09bec384429e6b24caf42cc9f Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 30 Nov 2021 15:16:03 -0800 Subject: [PATCH 07/11] Add integration testing --- .../samlib/resource_metadata_normalizer.py | 10 ++- .../local/invoke/invoke_cdk_templates.py | 59 +++++++++++++++ .../testdata/invoke/cdk/cdk_template.yaml | 73 +++++++++++++++++++ .../invoke/cdk/lambda_code/Dockerfile | 7 ++ .../testdata/invoke/cdk/lambda_code/app.js | 17 +++++ 5 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 tests/integration/local/invoke/invoke_cdk_templates.py create mode 100644 tests/integration/testdata/invoke/cdk/cdk_template.yaml create mode 100644 tests/integration/testdata/invoke/cdk/lambda_code/Dockerfile create mode 100644 tests/integration/testdata/invoke/cdk/lambda_code/app.js diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 2b146f381f..01e6ab8d90 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -3,7 +3,7 @@ """ import logging -import pathlib +from pathlib import Path RESOURCES_KEY = "Resources" PROPERTIES_KEY = "Properties" @@ -46,6 +46,8 @@ def normalize(template_dict): if asset_property == IMAGE_ASSET_PROPERTY: asset_metadata = ResourceMetadataNormalizer._extract_image_asset_metadata(resource_metadata) ResourceMetadataNormalizer._update_resource_image_asset_metadata(resource_metadata, asset_metadata) + # For image-type functions, the asset path is expected to be the name of the Docker image. + # When building, we set the name of the image to be the logical id of the function. asset_path = logical_id.lower() else: asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) @@ -102,10 +104,10 @@ def _extract_image_asset_metadata(metadata): metadata properties for image-type lambda function """ - asset_path = pathlib.Path(metadata.get(ASSET_PATH_METADATA_KEY, "")) - dockerfile_path = pathlib.Path(metadata.get(ASSET_DOCKERFILE_PATH_KEY), "") + asset_path = Path(metadata.get(ASSET_PATH_METADATA_KEY, "")) + dockerfile_path = Path(metadata.get(ASSET_DOCKERFILE_PATH_KEY), "") dockerfile, path_from_asset = dockerfile_path.stem, dockerfile_path.parent - dockerfile_context = str(pathlib.Path(asset_path.joinpath(path_from_asset))) + dockerfile_context = str(Path(asset_path.joinpath(path_from_asset))) return { SAM_METADATA_DOCKERFILE_KEY: dockerfile, SAM_METADATA_DOCKER_CONTEXT_KEY: dockerfile_context, diff --git a/tests/integration/local/invoke/invoke_cdk_templates.py b/tests/integration/local/invoke/invoke_cdk_templates.py new file mode 100644 index 0000000000..635a83cbc1 --- /dev/null +++ b/tests/integration/local/invoke/invoke_cdk_templates.py @@ -0,0 +1,59 @@ +import json +import docker +from docker.errors import APIError +from parameterized import parameterized + +from samcli import __version__ as version +from samcli.local.docker.lambda_image import RAPID_IMAGE_TAG_PREFIX +from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase +from samcli.lib.utils.architecture import X86_64 + + +class TestCDKSynthesizedTemplatesFunctions(InvokeIntegBase): + + template = "cdk/cdk_template.yaml" + functions = [ + "StandardFunctionConstructZipFunction", + "StandardFunctionConstructImageFunction", + "DockerImageFunctionConstruct", + ] + + @classmethod + def setUpClass(cls): + # Run sam build first to build the image functions + super(TestCDKSynthesizedTemplatesFunctions, cls).setUpClass() + build_command_list = super().get_build_command_list(cls, template_path=cls.template_path) + super().run_command(cls, command_list=build_command_list) + + @classmethod + def tearDownClass(cls) -> None: + # Remove images + docker_client = docker.from_env() + for function in cls.functions: + try: + docker_client.api.remove_image(f"{function.lower()}") + docker_client.api.remove_image(f"{function.lower()}:{RAPID_IMAGE_TAG_PREFIX}-{version}-{X86_64}") + except APIError: + pass + + @parameterized.expand(functions) + def test_build_and_invoke_image_function(self, function_name): + local_invoke_command_list = self.get_command_list( + function_to_invoke=function_name, template_path=self.template_path + ) + stdout, _, return_code = self.run_command(local_invoke_command_list) + + # Get the response without the sam-cli prompts that proceed it + response = json.loads(stdout.decode("utf-8").split("\n")[0]) + expected_response = json.loads('{"statusCode":200,"body":"{\\"message\\":\\"hello world\\"}"}') + + self.assertEqual(return_code, 0) + self.assertEqual(response, expected_response) + + @parameterized.expand(functions) + def test_invoke_with_utf8_event(self, function_name): + command_list = self.get_command_list( + function_name, template_path=self.template_path, event_path=self.event_utf8_path + ) + stdout, _, return_code = self.run_command(command_list) + self.assertEqual(return_code, 0) diff --git a/tests/integration/testdata/invoke/cdk/cdk_template.yaml b/tests/integration/testdata/invoke/cdk/cdk_template.yaml new file mode 100644 index 0000000000..0fc85d23fd --- /dev/null +++ b/tests/integration/testdata/invoke/cdk/cdk_template.yaml @@ -0,0 +1,73 @@ +Resources: + StandardFunctionConstructImageFunction: + Type: AWS::Lambda::Function + Properties: + Code: + ImageUri: + Fn::Join: + - "" + - - Ref: AWS::AccountId + - .dkr.ecr. + - Ref: AWS::Region + - "." + - Ref: AWS::URLSuffix + - /aws-cdk/assets:ec366e0c559122e6b653100637b3745e7dd2c7bc882572b66fd53f498cc06007 + PackageType: Image + Metadata: + aws:cdk:path: ApiCorsIssueStack/Fn/Resource + aws:asset:path: ./lambda_code + aws:asset:dockerfile-path: Dockerfile + aws:asset:property: Code.ImageUri + DockerImageFunctionConstruct: + Type: AWS::Lambda::Function + Properties: + Code: + ImageUri: + Fn::Join: + - "" + - - Ref: AWS::AccountId + - .dkr.ecr. + - Ref: AWS::Region + - "." + - Ref: AWS::URLSuffix + - /aws-cdk/assets:ec366e0c559122e6b653100637b3745e7dd2c7bc882572b66fd53f498cc06007 + PackageType: Image + Metadata: + aws:cdk:path: ApiCorsIssueStack/DockerFunc/Resource + aws:asset:path: ./lambda_code + aws:asset:dockerfile-path: Dockerfile + aws:asset:property: Code.ImageUri + StandardFunctionConstructZipFunction: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: + Ref: AssetParametersd993ee10bdd2d5f2054086eb58ff286f13672de94811036fc40c647e0e1b17c7S3BucketEA8808C3 + S3Key: + Fn::Join: + - "" + - - Fn::Select: + - 0 + - Fn::Split: + - "||" + - Ref: AssetParametersd993ee10bdd2d5f2054086eb58ff286f13672de94811036fc40c647e0e1b17c7S3VersionKeyF3B3F55C + - Fn::Select: + - 1 + - Fn::Split: + - "||" + - Ref: AssetParametersd993ee10bdd2d5f2054086eb58ff286f13672de94811036fc40c647e0e1b17c7S3VersionKeyF3B3F55C + Handler: app.lambdaHandler + Runtime: nodejs12.x + Metadata: + aws:cdk:path: ApiCorsIssueStack/Lambda/Resource + aws:asset:path: ./lambda_code + aws:asset:original-path: ./lambda_code + aws:asset:is-bundled: false + aws:asset:property: Code + CDKMetadata: + Type: AWS::CDK::Metadata + Properties: + Analytics: v2:deflate64:H4sIAAAAAAAA/02OzQrCMBCEn8V7um2oeFYrgtf4BDFdS/qTlWyqSMi721QETzPsN7OMhCjrGqrNXr+4MO1QGvII8Rq0GURDjoOfTRDN3Slkmr3B7BfQ2mDJJSHr7X89ZhNHPd1aDfE8O5NjufPzSVg9QVQ0rq+yJoHGF5oZA8OJzID+MukOD/mSGSh8ENtA/n3UvOTXlStelnbWdSkJRy1Cz+VT7kBWy6ierS387IKdENRXPzi+2EPxAAAA + Metadata: + aws:cdk:path: ApiCorsIssueStack/CDKMetadata/Default + Condition: CDKMetadataAvailable diff --git a/tests/integration/testdata/invoke/cdk/lambda_code/Dockerfile b/tests/integration/testdata/invoke/cdk/lambda_code/Dockerfile new file mode 100644 index 0000000000..417f6f34e0 --- /dev/null +++ b/tests/integration/testdata/invoke/cdk/lambda_code/Dockerfile @@ -0,0 +1,7 @@ +FROM public.ecr.aws/lambda/nodejs:14 + +# Assumes your function is named "app.js", and there is a package.json file in the app directory +COPY app.js ./ + +# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile) +CMD [ "app.lambdaHandler" ] diff --git a/tests/integration/testdata/invoke/cdk/lambda_code/app.js b/tests/integration/testdata/invoke/cdk/lambda_code/app.js new file mode 100644 index 0000000000..4294ca6fec --- /dev/null +++ b/tests/integration/testdata/invoke/cdk/lambda_code/app.js @@ -0,0 +1,17 @@ +let response; + +exports.lambdaHandler = async (event, context) => { + try { + response = { + 'statusCode': 200, + 'body': JSON.stringify({ + message: 'hello world', + }) + } + } catch (err) { + console.log(err); + return err; + } + + return response +}; From aa51982c72dcf08cea684099d3aa05ce564d13ff Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 30 Nov 2021 16:02:59 -0800 Subject: [PATCH 08/11] Fix tests --- .../local/invoke/invoke_cdk_templates.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/integration/local/invoke/invoke_cdk_templates.py b/tests/integration/local/invoke/invoke_cdk_templates.py index 635a83cbc1..c9b2e38a1e 100644 --- a/tests/integration/local/invoke/invoke_cdk_templates.py +++ b/tests/integration/local/invoke/invoke_cdk_templates.py @@ -1,5 +1,9 @@ import json +from pathlib import Path +from unittest import skipIf + import docker +import pytest from docker.errors import APIError from parameterized import parameterized @@ -7,11 +11,16 @@ from samcli.local.docker.lambda_image import RAPID_IMAGE_TAG_PREFIX from tests.integration.local.invoke.invoke_integ_base import InvokeIntegBase from samcli.lib.utils.architecture import X86_64 +from tests.testing_utils import IS_WINDOWS, RUNNING_ON_CI, CI_OVERRIDE +@skipIf( + ((IS_WINDOWS and RUNNING_ON_CI) and not CI_OVERRIDE), + "Skip build tests on windows when running in CI unless overridden", +) class TestCDKSynthesizedTemplatesFunctions(InvokeIntegBase): - template = "cdk/cdk_template.yaml" + template = Path("cdk/cdk_template.yaml") functions = [ "StandardFunctionConstructZipFunction", "StandardFunctionConstructImageFunction", @@ -21,6 +30,8 @@ class TestCDKSynthesizedTemplatesFunctions(InvokeIntegBase): @classmethod def setUpClass(cls): # Run sam build first to build the image functions + # We only need to create these images once, and we can reuse the for + # all tests since they don't change. Then we remove them after all tests are run. super(TestCDKSynthesizedTemplatesFunctions, cls).setUpClass() build_command_list = super().get_build_command_list(cls, template_path=cls.template_path) super().run_command(cls, command_list=build_command_list) @@ -37,6 +48,7 @@ def tearDownClass(cls) -> None: pass @parameterized.expand(functions) + @pytest.mark.flaky(reruns=3) def test_build_and_invoke_image_function(self, function_name): local_invoke_command_list = self.get_command_list( function_to_invoke=function_name, template_path=self.template_path @@ -51,6 +63,7 @@ def test_build_and_invoke_image_function(self, function_name): self.assertEqual(response, expected_response) @parameterized.expand(functions) + @pytest.mark.flaky(reruns=3) def test_invoke_with_utf8_event(self, function_name): command_list = self.get_command_list( function_name, template_path=self.template_path, event_path=self.event_utf8_path From 67c8b7046928b8d47dfd813c4376ae1168b898b0 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 30 Nov 2021 17:45:40 -0800 Subject: [PATCH 09/11] Isolate resources to enable test parallelism --- .../AWS_SAM_CLI_README | 1 + .../local/invoke/invoke_cdk_templates.py | 36 +++++++++++-------- 2 files changed, 22 insertions(+), 15 deletions(-) create mode 100644 build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README diff --git a/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README b/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README new file mode 100644 index 0000000000..f6f5ac40c6 --- /dev/null +++ b/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README @@ -0,0 +1 @@ +This layer contains dependencies of function MyFunction and automatically added by AWS SAM CLI command 'sam sync' \ No newline at end of file diff --git a/tests/integration/local/invoke/invoke_cdk_templates.py b/tests/integration/local/invoke/invoke_cdk_templates.py index c9b2e38a1e..4585bcd575 100644 --- a/tests/integration/local/invoke/invoke_cdk_templates.py +++ b/tests/integration/local/invoke/invoke_cdk_templates.py @@ -1,4 +1,5 @@ import json +import shutil from pathlib import Path from unittest import skipIf @@ -30,26 +31,33 @@ class TestCDKSynthesizedTemplatesFunctions(InvokeIntegBase): @classmethod def setUpClass(cls): # Run sam build first to build the image functions - # We only need to create these images once, and we can reuse the for - # all tests since they don't change. Then we remove them after all tests are run. + # We only need to create these images once + # We remove them after they are no longer used super(TestCDKSynthesizedTemplatesFunctions, cls).setUpClass() build_command_list = super().get_build_command_list(cls, template_path=cls.template_path) super().run_command(cls, command_list=build_command_list) - @classmethod - def tearDownClass(cls) -> None: - # Remove images + def tearDown(self) -> None: + # Tear down a unique image resource after it is finished being used docker_client = docker.from_env() - for function in cls.functions: - try: - docker_client.api.remove_image(f"{function.lower()}") - docker_client.api.remove_image(f"{function.lower()}:{RAPID_IMAGE_TAG_PREFIX}-{version}-{X86_64}") - except APIError: - pass + try: + to_remove = self.teardown_function_name + docker_client.api.remove_image(f"{to_remove.lower()}") + docker_client.api.remove_image(f"{to_remove.lower()}:{RAPID_IMAGE_TAG_PREFIX}-{version}-{X86_64}") + except (APIError, AttributeError): + pass + + try: + # We don't actually use the build dir so we don't care if it's removed before another process finishes + shutil.rmtree(str(Path().joinpath(".aws-sam"))) + except FileNotFoundError: + pass @parameterized.expand(functions) @pytest.mark.flaky(reruns=3) def test_build_and_invoke_image_function(self, function_name): + # Set the function name to be removed during teardown + self.teardown_function_name = function_name local_invoke_command_list = self.get_command_list( function_to_invoke=function_name, template_path=self.template_path ) @@ -62,11 +70,9 @@ def test_build_and_invoke_image_function(self, function_name): self.assertEqual(return_code, 0) self.assertEqual(response, expected_response) - @parameterized.expand(functions) - @pytest.mark.flaky(reruns=3) - def test_invoke_with_utf8_event(self, function_name): + def test_invoke_with_utf8_event(self): command_list = self.get_command_list( - function_name, template_path=self.template_path, event_path=self.event_utf8_path + "StandardFunctionConstructZipFunction", template_path=self.template_path, event_path=self.event_utf8_path ) stdout, _, return_code = self.run_command(command_list) self.assertEqual(return_code, 0) From 5431fc46564020efce9ca64311ca63d07a27ab3b Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Tue, 30 Nov 2021 17:47:24 -0800 Subject: [PATCH 10/11] Remove build dir --- build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README | 1 - 1 file changed, 1 deletion(-) delete mode 100644 build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README diff --git a/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README b/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README deleted file mode 100644 index f6f5ac40c6..0000000000 --- a/build_dir/MyFunctionbf3066d1DepLayer/AWS_SAM_CLI_README +++ /dev/null @@ -1 +0,0 @@ -This layer contains dependencies of function MyFunction and automatically added by AWS SAM CLI command 'sam sync' \ No newline at end of file From 7630b2e2f7a2b0ace72f5071548bab851c737533 Mon Sep 17 00:00:00 2001 From: Daniel Mil Date: Wed, 1 Dec 2021 09:33:57 -0800 Subject: [PATCH 11/11] Enable test on windows --- appveyor-windows-build-nodejs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor-windows-build-nodejs.yml b/appveyor-windows-build-nodejs.yml index 4c5a571f74..db1808cb86 100644 --- a/appveyor-windows-build-nodejs.yml +++ b/appveyor-windows-build-nodejs.yml @@ -71,6 +71,7 @@ test_script: - "venv\\Scripts\\activate" - "docker system prune -a -f" - "pytest -vv tests/integration/buildcmd/test_build_cmd.py -k TestBuildCommand_NodeFunctions" + - "pytest -vv tests/integration/local/invoke/invoke_cdk_templates.py -k TestCDKSynthesizedTemplatesFunctions" # Uncomment for RDP # on_finish: