From a830f5a46ae54862bd688b1cc2be17f06c523095 Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Thu, 3 Jan 2019 13:43:52 -0800 Subject: [PATCH 1/4] feat(cdk): Design for reading CDK Metadata embedded in a template --- .../template_generation_through_frameworks.md | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 designs/template_generation_through_frameworks.md diff --git a/designs/template_generation_through_frameworks.md b/designs/template_generation_through_frameworks.md new file mode 100644 index 0000000000..7e3392a8d0 --- /dev/null +++ b/designs/template_generation_through_frameworks.md @@ -0,0 +1,145 @@ +Support template generation through other Frameworks +==================================================== + +This is a design to capture how SAM CLI can support templates that are generated +from different frameworks, e.g. AWS Cloud Development Kit. + +Initially, the support will only be for processing the CDK metadata that is appended into a template. + +What is the problem? +-------------------- + +Customers have different ways to define their AWS Resources. As of writing (Jan. 2109), +SAM CLI supports the use case of defining an application in CloudFormation/SAM (a super +set of CloudFormation). These CloudFormation/SAM applications are written in `json` or `yaml` +and deployed through AWS CloudFormation. Frameworks like CDK offer customers an alternative +in how they define their applications. SAM CLI should support the ability to invoke functions +defined through these other frameworks to enable them to locally debug or manage their +applications. + +What will be changed? +--------------------- + +To start, we will add support for processing metadata from CDK applications: +SAM CLI will add a processing step on the templates it reads. This will consist of reading +the template and for each resource reading the metadata and replacing values as specified. + +In the future, we can support creating these templates from the different frameworks in a command directly within +SAM CLI but is out of scope in the initial implementation of support. + +Success criteria for the change +------------------------------- + +* Ability to invoke functions locally that was defined in AWS Cloud Development Kit (CDK). +* Process a template with CDK Metadata on a resource. + +Out-of-Scope +------------ + +* A command that will generate the template from the framework. +* Handling multiple stacks. +* Support for frameworks other than CDK. + +User Experience Walkthrough +--------------------------- + +### Customer using CDK + +A customer will use CDK to generate the template. This can be done by generating a template and saving it to a file: +`cdk synth > template.yaml`. Then will then be able to `sam local [invoke|start-api|start-lambda]` any +function they have defined [1]. + + +[1] Note: The cdk version must be greater than v0.21.0 as the metadata needed to parse is not appended on older versions. + + +Implementation +============== + +CLI Changes +----------- + +For the features currently in scope, there are no changes to the CLI interface. + +### Breaking Change + +No breaking changes + +Design +------ + +All the providers, which are used to get resources out of the template provided to the command, call +`SamBaseProvider.get_template(template_dict, parameter_overrides)` to get a normalized template. This function call is +responsible for taking a SAM template dictionary and returning a cleaned copy of the template where SAM plugins have +been run and parameter values have been substituted. Given the current scope of this call, expanding it to also normalize +metadata, seems reasonable. We will expand `SamBaseProvider.get_tempalte()` to call a `CdkTemplateNormalizer` class +that will be responsible for understanding the metadata and normalizing the template with respect to the metadata. + +```python + +class CdkTemplateNormalizer(object): + + @staticmethod + def normalize(template_dict): + for resource in template_dict.get('Resources'): + if 'Metadata' in resource: + CdkTemplateNormalizer.replace_property(key, value) +``` + +`.samrc` Changes +---------------- + +N/A + +Security +-------- + +*Tip: How does this change impact security? Answer the following +questions to help answer this question better:* + +**What new dependencies (libraries/cli) does this change require?** + +None + +**What other Docker container images are you using?** + +None + +**Are you creating a new HTTP endpoint? If so explain how it will be +created & used** + +No + +**Are you connecting to a remote API? If so explain how is this +connection secured** + +No + +**Are you reading/writing to a temporary folder? If so, what is this +used for and when do you clean up?** + +No + +**How do you validate new .samrc configuration?** + +N/A + +Documentation Changes +--------------------- + +* Blog or Documentation that explains how you can define an application in CDK and use SAM CLI to test/invoke + +Open Issues +----------- + +Task Breakdown +-------------- + +- \[x\] Send a Pull Request with this design document +- \[ \] Build the command line interface +- \[ \] Build the underlying library +- \[ \] Unit tests +- \[ \] Functional Tests +- \[ \] Integration tests +- \[ \] Run all tests on Windows +- \[ \] Update documentation From f3e233ba0939db90e5d1f3af09bf99810aa23a0c Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Fri, 4 Jan 2019 13:28:01 -0800 Subject: [PATCH 2/4] Update design to center around Resource Metadata instead of a specific framework --- ...rks.md => resource_metadata_overriding.md} | 49 ++++++++++++++----- 1 file changed, 37 insertions(+), 12 deletions(-) rename designs/{template_generation_through_frameworks.md => resource_metadata_overriding.md} (69%) diff --git a/designs/template_generation_through_frameworks.md b/designs/resource_metadata_overriding.md similarity index 69% rename from designs/template_generation_through_frameworks.md rename to designs/resource_metadata_overriding.md index 7e3392a8d0..91dafdb20f 100644 --- a/designs/template_generation_through_frameworks.md +++ b/designs/resource_metadata_overriding.md @@ -1,10 +1,11 @@ -Support template generation through other Frameworks -==================================================== +Understand Resource Level Metadata +================================== This is a design to capture how SAM CLI can support templates that are generated from different frameworks, e.g. AWS Cloud Development Kit. -Initially, the support will only be for processing the CDK metadata that is appended into a template. +Initially, the support will only be for processing Resource Metadata within the template, which enables support for +customers using AWS Cloud Development Kit (CDK). What is the problem? -------------------- @@ -20,9 +21,9 @@ applications. What will be changed? --------------------- -To start, we will add support for processing metadata from CDK applications: +To start, we will add support for processing Resource Metadata that is embedded into the template: SAM CLI will add a processing step on the templates it reads. This will consist of reading -the template and for each resource reading the metadata and replacing values as specified. +the template and for each resource reading the Metadata and replacing values as specified. In the future, we can support creating these templates from the different frameworks in a command directly within SAM CLI but is out of scope in the initial implementation of support. @@ -30,19 +31,19 @@ SAM CLI but is out of scope in the initial implementation of support. Success criteria for the change ------------------------------- -* Ability to invoke functions locally that was defined in AWS Cloud Development Kit (CDK). +* Ability to invoke functions locally that contain Metadata on a Resource * Process a template with CDK Metadata on a resource. Out-of-Scope ------------ * A command that will generate the template from the framework. -* Handling multiple stacks. -* Support for frameworks other than CDK. User Experience Walkthrough --------------------------- +CDK is a framework that appends this Metadata to Resources within a template and will use this as an example. + ### Customer using CDK A customer will use CDK to generate the template. This can be done by generating a template and saving it to a file: @@ -72,18 +73,42 @@ All the providers, which are used to get resources out of the template provided `SamBaseProvider.get_template(template_dict, parameter_overrides)` to get a normalized template. This function call is responsible for taking a SAM template dictionary and returning a cleaned copy of the template where SAM plugins have been run and parameter values have been substituted. Given the current scope of this call, expanding it to also normalize -metadata, seems reasonable. We will expand `SamBaseProvider.get_tempalte()` to call a `CdkTemplateNormalizer` class +metadata, seems reasonable. We will expand `SamBaseProvider.get_tempalte()` to call a `TemplateMetadataNormalizer` class that will be responsible for understanding the metadata and normalizing the template with respect to the metadata. -```python +Template snippet that contains the metadata SAM CLI will parse and understand. + +```yaml + +Resources: + MyFunction: + Type: AWS::Lambda::Function + Properties: + Code: + S3Bucket: mybucket + S3Key: myKey + ... + Metadata: + aws:asset:path: '/path/to/function/code' + aws:asset:property: 'Code' +``` -class CdkTemplateNormalizer(object): +The two keys we will recognize are `aws:asset:path` and `aws:asset:property`. `aws:asset:path`'s value will be the path +to the code, files, etc that are help on the machine, while `aws:asset:property` is the Property of the Resource that +needs to be replaced. So in the example above, the `Code` Property will be replaced with `/path/to/function/code`. + +Below algorithm to do this Metadata Normalization on the template. + +```python +class TemplateMetadataNormalizer(object): @staticmethod def normalize(template_dict): for resource in template_dict.get('Resources'): if 'Metadata' in resource: - CdkTemplateNormalizer.replace_property(key, value) + asset_property = resource.get('Metadata').get('aws:asset:property') + asset_path = resource.get('Metadata').get('aws:asset:path') + TemplateMetadataNormalizer.replace_property(asset_property, asset_path) ``` `.samrc` Changes From b056a5b77e4f50e11c82537c4c1e9c6a9161a961 Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Mon, 7 Jan 2019 07:12:27 -0800 Subject: [PATCH 3/4] Implementation and updated design --- designs/resource_metadata_overriding.md | 8 +- .../commands/local/lib/sam_base_provider.py | 5 +- .../samlib/resource_metadata_normalizer.py | 54 +++++++ .../local/invoke/test_integrations_cli.py | 11 ++ .../different_code_location/__init__.py | 0 .../invoke/different_code_location/main.py | 2 + .../integration/testdata/invoke/template.yml | 9 ++ .../local/lib/test_sam_base_provider.py | 10 +- .../test_resource_metadata_normalizer.py | 138 ++++++++++++++++++ 9 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 samcli/lib/samlib/resource_metadata_normalizer.py create mode 100644 tests/integration/testdata/invoke/different_code_location/__init__.py create mode 100644 tests/integration/testdata/invoke/different_code_location/main.py create mode 100644 tests/unit/lib/samlib/test_resource_metadata_normalizer.py diff --git a/designs/resource_metadata_overriding.md b/designs/resource_metadata_overriding.md index 91dafdb20f..b814a851b1 100644 --- a/designs/resource_metadata_overriding.md +++ b/designs/resource_metadata_overriding.md @@ -73,7 +73,7 @@ All the providers, which are used to get resources out of the template provided `SamBaseProvider.get_template(template_dict, parameter_overrides)` to get a normalized template. This function call is responsible for taking a SAM template dictionary and returning a cleaned copy of the template where SAM plugins have been run and parameter values have been substituted. Given the current scope of this call, expanding it to also normalize -metadata, seems reasonable. We will expand `SamBaseProvider.get_tempalte()` to call a `TemplateMetadataNormalizer` class +metadata, seems reasonable. We will expand `SamBaseProvider.get_tempalte()` to call a `ResourceMetadataNormalizer` class that will be responsible for understanding the metadata and normalizing the template with respect to the metadata. Template snippet that contains the metadata SAM CLI will parse and understand. @@ -94,13 +94,13 @@ Resources: ``` The two keys we will recognize are `aws:asset:path` and `aws:asset:property`. `aws:asset:path`'s value will be the path -to the code, files, etc that are help on the machine, while `aws:asset:property` is the Property of the Resource that +to the code, files, etc that are on the machine, while `aws:asset:property` is the Property of the Resource that needs to be replaced. So in the example above, the `Code` Property will be replaced with `/path/to/function/code`. Below algorithm to do this Metadata Normalization on the template. ```python -class TemplateMetadataNormalizer(object): +class ResourceMetadataNormalizer(object): @staticmethod def normalize(template_dict): @@ -108,7 +108,7 @@ class TemplateMetadataNormalizer(object): if 'Metadata' in resource: asset_property = resource.get('Metadata').get('aws:asset:property') asset_path = resource.get('Metadata').get('aws:asset:path') - TemplateMetadataNormalizer.replace_property(asset_property, asset_path) + ResourceMetadataNormalizer.replace_property(asset_property, asset_path) ``` `.samrc` Changes diff --git a/samcli/commands/local/lib/sam_base_provider.py b/samcli/commands/local/lib/sam_base_provider.py index 7b26f3dbb5..bbf4d6381b 100644 --- a/samcli/commands/local/lib/sam_base_provider.py +++ b/samcli/commands/local/lib/sam_base_provider.py @@ -4,10 +4,12 @@ import logging -from samcli.lib.samlib.wrapper import SamTranslatorWrapper from samtranslator.intrinsics.resolver import IntrinsicsResolver from samtranslator.intrinsics.actions import RefAction +from samcli.lib.samlib.wrapper import SamTranslatorWrapper +from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer + LOG = logging.getLogger(__name__) @@ -60,6 +62,7 @@ def get_template(template_dict, parameter_overrides=None): template_dict = SamTranslatorWrapper(template_dict).run_plugins() template_dict = SamBaseProvider._resolve_parameters(template_dict, parameter_overrides) + ResourceMetadataNormalizer.normalize(template_dict) return template_dict @staticmethod diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py new file mode 100644 index 0000000000..81f78286ed --- /dev/null +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -0,0 +1,54 @@ +""" +Class that Normalizes a Template based on Resource Metadata +""" + +RESOURCES_KEY = "Resources" +PROPERTIES_KEY = "Properties" +METADATA_KEY = "Metadata" +ASSET_PATH_METADATA_KEY = "aws:asset:path" +ASSET_PROPERTY_METADATA_KEY = "aws:asset:property" + + +class ResourceMetadataNormalizer(object): + + @staticmethod + def normalize(template_dict): + """ + Normalize all Resources in the template with the Metadata Key on the resource. + + This method will mutate the template + + Parameters + ---------- + template_dict dict + Dictionary representing the template + + """ + resources = template_dict.get(RESOURCES_KEY, {}) + + for _, resource in resources.items(): + resource_metadata = resource.get(METADATA_KEY, {}) + asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) + asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY) + + ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource) + + @staticmethod + def _replace_property(property_key, property_value, resource): + """ + Replace a property with an asset on a given resource + + This method will mutate the template + + Parameters + ---------- + property str + The property to replace on the resource + property_value str + The new value of the property + resource dict + Dictionary representing the Resource to change + + """ + if property_value and property_key: + resource.get(PROPERTIES_KEY, {})[property_key] = property_value diff --git a/tests/integration/local/invoke/test_integrations_cli.py b/tests/integration/local/invoke/test_integrations_cli.py index a9c383e10b..d39e400a8f 100644 --- a/tests/integration/local/invoke/test_integrations_cli.py +++ b/tests/integration/local/invoke/test_integrations_cli.py @@ -36,6 +36,17 @@ def test_invoke_returncode_is_zero(self): self.assertEquals(return_code, 0) + def test_function_with_metadata(self): + command_list = self.get_command_list("FunctionWithMetadata", + template_path=self.template_path, + no_event=True) + + process = Popen(command_list, stdout=PIPE) + process.wait() + process_stdout = b"".join(process.stdout.readlines()).strip() + + self.assertEquals(process_stdout.decode('utf-8'), '"Hello World in a different dir"') + def test_invoke_returns_execpted_results(self): command_list = self.get_command_list("HelloWorldServerlessFunction", template_path=self.template_path, diff --git a/tests/integration/testdata/invoke/different_code_location/__init__.py b/tests/integration/testdata/invoke/different_code_location/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/invoke/different_code_location/main.py b/tests/integration/testdata/invoke/different_code_location/main.py new file mode 100644 index 0000000000..7422a87c67 --- /dev/null +++ b/tests/integration/testdata/invoke/different_code_location/main.py @@ -0,0 +1,2 @@ +def echo_hello_world(event, context): + return "Hello World in a different dir" diff --git a/tests/integration/testdata/invoke/template.yml b/tests/integration/testdata/invoke/template.yml index 8a6e799f2f..3a3d1c51d9 100644 --- a/tests/integration/testdata/invoke/template.yml +++ b/tests/integration/testdata/invoke/template.yml @@ -88,6 +88,15 @@ Resources: Timeout: Ref: DefaultTimeout + FunctionWithMetadata: + Type: AWS::Serverless::Function + Properties: + Runtime: python3.6 + Handler: main.echo_hello_world + Metadata: + aws:asset:property: CodeUri + aws:asset:path: ./different_code_location + EchoEnvWithParameters: Type: AWS::Serverless::Function Properties: diff --git a/tests/unit/commands/local/lib/test_sam_base_provider.py b/tests/unit/commands/local/lib/test_sam_base_provider.py index 001a217058..c9d870e95a 100644 --- a/tests/unit/commands/local/lib/test_sam_base_provider.py +++ b/tests/unit/commands/local/lib/test_sam_base_provider.py @@ -178,11 +178,18 @@ def test_must_skip_empty_template(self): class TestSamBaseProvider_get_template(TestCase): + @patch("samcli.commands.local.lib.sam_base_provider.ResourceMetadataNormalizer") @patch("samcli.commands.local.lib.sam_base_provider.SamTranslatorWrapper") @patch.object(SamBaseProvider, "_resolve_parameters") - def test_must_run_translator_plugins(self, resolve_params_mock, SamTranslatorWrapperMock): + def test_must_run_translator_plugins(self, + resolve_params_mock, + SamTranslatorWrapperMock, + resource_metadata_normalizer_patch): translator_instance = SamTranslatorWrapperMock.return_value = Mock() + parameter_resolved_template = {"Key": "Value", "Parameter": "Resolved"} + resolve_params_mock.return_value = parameter_resolved_template + template = {"Key": "Value"} overrides = {'some': 'value'} @@ -191,3 +198,4 @@ def test_must_run_translator_plugins(self, resolve_params_mock, SamTranslatorWra SamTranslatorWrapperMock.assert_called_once_with(template) translator_instance.run_plugins.assert_called_once() resolve_params_mock.assert_called_once() + resource_metadata_normalizer_patch.normalize.assert_called_once_with(parameter_resolved_template) diff --git a/tests/unit/lib/samlib/test_resource_metadata_normalizer.py b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py new file mode 100644 index 0000000000..bd4a9caffa --- /dev/null +++ b/tests/unit/lib/samlib/test_resource_metadata_normalizer.py @@ -0,0 +1,138 @@ +from unittest import TestCase + +from samcli.lib.samlib.resource_metadata_normalizer import ResourceMetadataNormalizer + + +class TestResourceMeatadataNormalizer(TestCase): + + def test_replace_property_with_path(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": "some value" + }, + "Metadata": { + "aws:asset:path": "new path", + "aws:asset:property": "Code" + } + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("new path", template_data['Resources']['Function1']['Properties']['Code']) + + def test_replace_all_resources_that_contain_metadata(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": "some value" + }, + "Metadata": { + "aws:asset:path": "new path", + "aws:asset:property": "Code" + } + }, + "Resource2": { + "Properties": { + "SomeRandomProperty": "some value" + }, + "Metadata": { + "aws:asset:path": "super cool path", + "aws:asset:property": "SomeRandomProperty" + } + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("new path", template_data['Resources']['Function1']['Properties']['Code']) + self.assertEqual("super cool path", template_data['Resources']['Resource2']['Properties']['SomeRandomProperty']) + + def test_tempate_without_metadata(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": "some value" + } + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("some value", template_data['Resources']['Function1']['Properties']['Code']) + + def test_template_without_asset_property(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": "some value" + }, + "Metadata": { + "aws:asset:path": "new path", + } + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("some value", template_data['Resources']['Function1']['Properties']['Code']) + + def test_tempalte_without_asset_path(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": "some value" + }, + "Metadata": { + "aws:asset:property": "Code" + } + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("some value", template_data['Resources']['Function1']['Properties']['Code']) + + def test_template_with_empty_metadata(self): + template_data = { + "Resources": { + "Function1": { + "Properties": { + "Code": "some value" + }, + "Metadata": {} + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("some value", template_data['Resources']['Function1']['Properties']['Code']) + + def test_replace_of_property_that_does_not_exist(self): + template_data = { + "Resources": { + "Function1": { + "Properties": {}, + "Metadata": { + "aws:asset:path": "new path", + "aws:asset:property": "Code" + } + } + } + } + + ResourceMetadataNormalizer.normalize(template_data) + + self.assertEqual("new path", template_data['Resources']['Function1']['Properties']['Code']) From 933ee7cd10abdc494f02919aabd705839317e436 Mon Sep 17 00:00:00 2001 From: Jacob Fuss Date: Mon, 7 Jan 2019 11:03:29 -0800 Subject: [PATCH 4/4] Add warning message when only one of aws:asset:* is provided --- .../lib/samlib/resource_metadata_normalizer.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index 81f78286ed..945246bd81 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -2,12 +2,16 @@ Class that Normalizes a Template based on Resource Metadata """ +import logging + RESOURCES_KEY = "Resources" PROPERTIES_KEY = "Properties" METADATA_KEY = "Metadata" ASSET_PATH_METADATA_KEY = "aws:asset:path" ASSET_PROPERTY_METADATA_KEY = "aws:asset:property" +LOG = logging.getLogger(__name__) + class ResourceMetadataNormalizer(object): @@ -26,15 +30,15 @@ def normalize(template_dict): """ resources = template_dict.get(RESOURCES_KEY, {}) - for _, resource in resources.items(): + for logical_id, resource in resources.items(): resource_metadata = resource.get(METADATA_KEY, {}) asset_path = resource_metadata.get(ASSET_PATH_METADATA_KEY) asset_property = resource_metadata.get(ASSET_PROPERTY_METADATA_KEY) - ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource) + ResourceMetadataNormalizer._replace_property(asset_property, asset_path, resource, logical_id) @staticmethod - def _replace_property(property_key, property_value, resource): + def _replace_property(property_key, property_value, resource, logical_id): """ Replace a property with an asset on a given resource @@ -48,7 +52,12 @@ def _replace_property(property_key, property_value, resource): The new value of the property resource dict Dictionary representing the Resource to change + logical_id str + LogicalId of the Resource """ - if property_value and property_key: + if property_key and property_value: resource.get(PROPERTIES_KEY, {})[property_key] = property_value + elif property_key or property_value: + LOG.info("WARNING: Ignoring Metadata for Resource %s. Metadata contains only aws:asset:path or " + "aws:assert:property but not both", logical_id)