From 30719a41b7dacb4528e98fe9674d90ea930187f9 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Tue, 29 Jun 2021 13:21:39 -0700 Subject: [PATCH 01/38] Cdk support package and deploy (#352) * Refactor project type click option * Refactor IAC helper * Update callbacks handling --cdk-app and --template * Add methods for stack in iac interface; Update CFN plugin to link image assets * Refactor option validations and update package cli interface * Update commands to include iac option validations * Fix iac validation * sam package for CDK * sam package & deploy for CDK * Update option validations to deal with guided deploy * Update test for guided deploy for CDK * Upgrade lambda builder * chore: Update aws_lambda_builders to 1.4.0 (#2903) * chore: Update aws_lambda_builders to 1.4.0 * Update integration tests for new maven behavior * Add integ test for PEP 600 tags * Update to update asset parameter after pacakage * Update iac cdk unit tests * Update iac cdk unit tests * resolve PR comments * resolve PR comments Co-authored-by: _sam <3804518+aahung@users.noreply.github.com> Co-authored-by: Mohamed Elasmar --- requirements/base.txt | 2 +- requirements/reproducible-linux.txt | 8 +- samcli/commands/_utils/iac_validations.py | 75 ++ samcli/commands/_utils/options.py | 111 +-- samcli/commands/_utils/resources.py | 5 + samcli/commands/_utils/template.py | 6 +- samcli/commands/build/command.py | 6 +- samcli/commands/deploy/command.py | 31 +- samcli/commands/deploy/deploy_context.py | 4 +- samcli/commands/deploy/guided_context.py | 73 +- samcli/commands/local/invoke/cli.py | 8 +- samcli/commands/local/start_api/cli.py | 8 +- samcli/commands/local/start_lambda/cli.py | 8 +- samcli/commands/package/command.py | 47 +- samcli/commands/package/package_context.py | 21 +- samcli/commands/package/utils.py | 25 + samcli/commands/package/validations.py | 43 + .../image_repository_validation.py | 21 +- samcli/lib/iac/cdk/cloud_assembly.py | 2 +- samcli/lib/iac/cdk/plugin.py | 160 ++-- samcli/lib/iac/cfn_iac.py | 53 +- samcli/lib/iac/interface.py | 116 ++- samcli/lib/iac/utils/helpers.py | 52 +- samcli/lib/package/artifact_exporter.py | 72 +- samcli/lib/package/packageable_resources.py | 138 ++-- samcli/lib/package/utils.py | 19 +- samcli/lib/providers/sam_base_provider.py | 18 +- samcli/lib/providers/sam_stack_provider.py | 22 +- .../samlib/resource_metadata_normalizer.py | 1 + tests/integration/buildcmd/test_build_cmd.py | 115 +-- .../buildcmd/PythonPEP600/__init__.py | 0 .../testdata/buildcmd/PythonPEP600/main.py | 19 + .../buildcmd/PythonPEP600/requirements.txt | 6 + .../commands/_utils/test_iac_validations.py | 153 ++++ tests/unit/commands/_utils/test_options.py | 64 ++ tests/unit/commands/deploy/test_command.py | 134 ++-- .../commands/deploy/test_guided_context.py | 242 +++--- tests/unit/commands/package/test_command.py | 18 + .../commands/package/test_package_context.py | 22 +- .../unit/commands/samconfig/test_samconfig.py | 147 ++-- .../test_image_repository_validation.py | 230 +++--- tests/unit/lib/iac/cdk/test_cloud_assembly.py | 15 + .../root-stack.template.normalized.json | 2 +- tests/unit/lib/iac/cdk/test_plugin.py | 746 +++++++++++++++++- tests/unit/lib/iac/test_interface.py | 323 ++++++++ .../lib/package/test_artifact_exporter.py | 670 ++++++++++------ 46 files changed, 3081 insertions(+), 980 deletions(-) create mode 100644 samcli/commands/_utils/iac_validations.py create mode 100644 samcli/commands/package/utils.py create mode 100644 samcli/commands/package/validations.py create mode 100644 tests/integration/testdata/buildcmd/PythonPEP600/__init__.py create mode 100644 tests/integration/testdata/buildcmd/PythonPEP600/main.py create mode 100644 tests/integration/testdata/buildcmd/PythonPEP600/requirements.txt create mode 100644 tests/unit/commands/_utils/test_iac_validations.py create mode 100644 tests/unit/lib/iac/test_interface.py diff --git a/requirements/base.txt b/requirements/base.txt index 839399daa2..e52f6d63e1 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -12,6 +12,6 @@ docker~=4.2.0 dateparser~=0.7 requests==2.23.0 serverlessrepo==0.1.10 -aws_lambda_builders==1.3.0 +aws_lambda_builders==1.4.0 tomlkit==0.7.0 watchdog==0.10.3 diff --git a/requirements/reproducible-linux.txt b/requirements/reproducible-linux.txt index aee4688122..abaf26ae99 100644 --- a/requirements/reproducible-linux.txt +++ b/requirements/reproducible-linux.txt @@ -12,10 +12,10 @@ attrs==20.3.0 \ --hash=sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6 \ --hash=sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700 # via jsonschema -aws-lambda-builders==1.3.0 \ - --hash=sha256:1e1d66173f19a1c40e3db96588bdea07a5bd16cd98d9d20ea2088dca2e7299d7 \ - --hash=sha256:a84521eea781967eb0a146c746b27195d29e7ac0d896d0dfb27608c53400eebb \ - --hash=sha256:d19724b51939bf9a8d78364b9ad63d8a7aa958942a963e19e6eff096c02b05d2 +aws-lambda-builders==1.4.0 \ + --hash=sha256:3f885433bb71bae653b520e3cf4c31fe5f5b977cb770d42c631af155cd60fd2b \ + --hash=sha256:5d4e4ecb3d3290f0eec1f62b7b0d9d6b91160ae71447d95899eede392d05f75f \ + --hash=sha256:d32f79cf67b189a7598793f69797f284b2eb9a9fada562175b1e854187f95aed # via aws-sam-cli (setup.py) aws-sam-translator==1.35.0 \ --hash=sha256:2f8904fd4a631752bc441a8fd928c444ed98ceb86b94d25ed7b84982e2eff1cd \ diff --git a/samcli/commands/_utils/iac_validations.py b/samcli/commands/_utils/iac_validations.py new file mode 100644 index 0000000000..3008ecfc6c --- /dev/null +++ b/samcli/commands/_utils/iac_validations.py @@ -0,0 +1,75 @@ +""" +Option validation for IAC Plugin +Use as decorator and place the decorator after: +1. all CLI options have been processed. +2. iac plugin has been injected +""" +import functools +import logging +import click + +from samcli.lib.iac.interface import ProjectTypes + + +LOG = logging.getLogger(__name__) + + +def iac_options_validation(require_stack=False): + """ + Wrapper validation function that will run after cli parameters have been loaded + and iac plugin has been injected. Validations vary based on project type. + + :param require_stack: a boolean flag to set whether --stack-name is required or not + :return: Click command function after validation + """ + + def inner(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + ctx = click.get_current_context() + selected_project_type = ctx.params.get("project_type") + + project_type_options_map = { + ProjectTypes.CFN.value: {}, + ProjectTypes.CDK.value: { + "cdk_app": "--cdk-app", + "cdk_context": "--cdk-context", + }, + } + + # validate if any option is used for the wrong project type + for project_type, options in project_type_options_map.items(): + if project_type == selected_project_type: + continue + for param_name, option_name in options.items(): + if ctx.params.get(param_name) is not None and ctx.params.get(param_name) != (): + raise click.BadOptionUsage( + option_name=option_name, + ctx=ctx, + message=f"Option '{option_name}' cannot be used for Project Type '{selected_project_type}'", + ) + + project_types_requring_stack_check = [ProjectTypes.CDK.value] + project = kwargs.get("project") + guided = ctx.params.get("guided", False) or ctx.params.get("g", False) + stack_name = ctx.params.get("stack_name") + if selected_project_type in project_types_requring_stack_check and project and require_stack and not guided: + if ctx.params.get("stack_name") is not None: + if project.find_stack_by_name(stack_name) is None: + raise click.BadOptionUsage( + option_name="--stack-name", + ctx=ctx, + message=f"Stack with stack name '{stack_name}' not found.", + ) + elif len(project.stacks) > 1: + raise click.BadOptionUsage( + option_name="--stack-name", + ctx=ctx, + message="More than one stack found. Use '--stack-name' to specify the stack.", + ) + + return func(*args, **kwargs) + + return wrapped + + return inner diff --git a/samcli/commands/_utils/options.py b/samcli/commands/_utils/options.py index 377c4c6c73..712da5dcc1 100644 --- a/samcli/commands/_utils/options.py +++ b/samcli/commands/_utils/options.py @@ -9,10 +9,9 @@ import click from click.types import FuncParamType -from samcli.commands._utils.template import get_template_data, TemplateNotFoundException from samcli.cli.types import CfnParameterOverridesType, CfnMetadataType, CfnTags, SigningProfilesOptionType +from samcli.commands._utils.template import get_template_data, TemplateNotFoundException, get_template_artifacts_format from samcli.commands._utils.custom_options.option_nargs import OptionNargs -from samcli.commands._utils.template import get_template_artifacts_format from samcli.lib.iac.interface import ProjectTypes _TEMPLATE_OPTION_DEFAULT_VALUE = "template.[yaml|yml]" @@ -26,7 +25,6 @@ def get_or_default_template_file_name(ctx, param, provided_value, include_build) Default value for the template file name option is more complex than what Click can handle. This method either returns user provided file name or one of the two default options (template.yaml/template.yml) depending on the file that exists - :param ctx: Click Context :param param: Param name :param provided_value: Value provided by Click. It could either be the default value or provided by user. @@ -359,59 +357,61 @@ def notification_arns_click_option(): ) -def project_type_click_option(f): - click.option( +def notification_arns_override_option(f): + return notification_arns_click_option()(f) + + +def _space_separated_list_func_type(value): + if isinstance(value, str): + return value.split(" ") + if isinstance(value, (list, tuple)): + return value + raise ValueError() + + +_space_separated_list_func_type.__name__ = "LIST" + + +def project_type_click_option(include_build=True): + return click.option( "--project-type", help="Project Type", - callback=partial(determine_project_type, include_build=True), + is_eager=True, + callback=partial(project_type_callback, include_build=include_build), type=click.Choice(ProjectTypes.__members__, case_sensitive=False), - )(f) - - return f - + ) -def cdk_click_options(f): - options = [ - click.option( - "--cdk-app", - required=False, - default=None, - help="Executable for your CDK app (e.g. node bin/my-app.js)" - "or the path of cloud assembly (e.g. ./cdk.out).", - ), - click.option( - "--cdk-context", - required=False, - default=[], - multiple=True, - help="Runtime context in key-value pairs for your CDK app." - "e.g. sam build --cdk-context Key1=Value1 --cdk-context Key2=Value2", - ), - ] - for option in options: - option(f) +def project_type_callback(ctx, param, provided_value, include_build): + """ + Callback for `--project-type` + """ + detected_project_type = determine_project_type(include_build) + if provided_value is not None: + if provided_value != detected_project_type: + raise click.BadOptionUsage( + option_name=param.name, + ctx=ctx, + message=f"It seems your project type is {detected_project_type}. " + f"However, you specified {provided_value} in --project-type", + ) + LOG.debug("Using customized project type %s.", provided_value) + return provided_value - return f + return detected_project_type -def determine_project_type(ctx, param, provided_value, include_build): +def determine_project_type(include_build): """ Determine the type of IaC Project to use. If SAM template file exists, project_type will be “CFN” Else if cdk.json exists in the root directory, project_type will be “CDK” Else, project_type will be “CFN” - :param ctx: Click Context - :param param: Param name - :param provided_value: Value provided by Click. It could either be the default value or provided by user. :param include_build: A boolean to set whether to search build template or not. :return: Project type """ LOG.debug("Determining project type...") - if provided_value: - LOG.debug("Using customized project type %s.", provided_value) - return provided_value if find_cfn_template(include_build): LOG.debug("The project is a CFN project.") @@ -451,16 +451,29 @@ def find_cdk_file(): return find_in_paths(search_paths) -def notification_arns_override_option(f): - return notification_arns_click_option()(f) - - -def _space_separated_list_func_type(value): - if isinstance(value, str): - return value.split(" ") - if isinstance(value, (list, tuple)): - return value - raise ValueError() +def cdk_click_options(f): + """ + click options for specifying cdk related behavior + """ + options = [ + click.option( + "--cdk-app", + required=False, + default=None, + is_eager=True, + help="Executable for your CDK app (e.g. node bin/my-app.js)" + "or the path of cloud assembly (e.g. ./cdk.out).", + ), + click.option( + "--cdk-context", + required=False, + multiple=True, + help="Runtime context in key-value pairs for your CDK app." + "e.g. sam build --cdk-context Key1=Value1 --cdk-context Key2=Value2", + ), + ] + for option in options: + option(f) -_space_separated_list_func_type.__name__ = "LIST" + return f diff --git a/samcli/commands/_utils/resources.py b/samcli/commands/_utils/resources.py index d3b2a18be3..af1629f8d1 100644 --- a/samcli/commands/_utils/resources.py +++ b/samcli/commands/_utils/resources.py @@ -52,6 +52,11 @@ AWS_LAMBDA_FUNCTION: ["Code"], } +NESTED_STACKS_RESOURCES = { + AWS_SERVERLESS_APPLICATION: "Location", + AWS_CLOUDFORMATION_STACK: "TemplateURL", +} + def get_packageable_resource_paths(): """ diff --git a/samcli/commands/_utils/template.py b/samcli/commands/_utils/template.py index 09d1c36e59..1d995023be 100644 --- a/samcli/commands/_utils/template.py +++ b/samcli/commands/_utils/template.py @@ -12,7 +12,7 @@ from botocore.utils import set_value_from_jmespath from samcli.commands.exceptions import UserException -from samcli.lib.iac.interface import S3Asset +from samcli.lib.iac.interface import S3Asset, Stack as IacStack from samcli.lib.samlib.resource_metadata_normalizer import METADATA_KEY, ASSET_PATH_METADATA_KEY from samcli.lib.utils.packagetype import ZIP from samcli.yamlhelper import yaml_parse, yaml_dump @@ -114,6 +114,10 @@ def move_template( if output_format == TemplateFormat.YAML: fp.write(yaml_dump(modified_template)) elif output_format == TemplateFormat.JSON: + if isinstance(modified_template, IacStack): + # IacStack inherits MutableMapping, which is not JSON serializable. + # We need to convert it into a dict obj + modified_template = modified_template.as_dict() fp.write(json.dumps(modified_template, indent=4)) diff --git a/samcli/commands/build/command.py b/samcli/commands/build/command.py index 2ce944b603..d5c93d0584 100644 --- a/samcli/commands/build/command.py +++ b/samcli/commands/build/command.py @@ -8,6 +8,7 @@ import click from samcli.cli.context import Context +from samcli.commands._utils.iac_validations import iac_options_validation from samcli.commands._utils.options import ( template_option_without_build, docker_common_options, @@ -173,7 +174,7 @@ "requests=1.x and the latest request module version changes from 1.1 to 1.2, " "SAM will not pull the latest version until you run a non-cached build.", ) -@project_type_click_option +@project_type_click_option(include_build=False) @cdk_click_options @template_option_without_build @parameter_override_option @@ -181,9 +182,10 @@ @cli_framework_options @aws_creds_options @click.argument("resource_logical_id", required=False) +@inject_iac_plugin(with_build=False) +@iac_options_validation(require_stack=False) @pass_context @track_command -@inject_iac_plugin(with_build=False) @check_newer_version @print_cmdline_args def cli( diff --git a/samcli/commands/deploy/command.py b/samcli/commands/deploy/command.py index 371dc61c4d..8209dd060a 100644 --- a/samcli/commands/deploy/command.py +++ b/samcli/commands/deploy/command.py @@ -8,9 +8,10 @@ from samcli.cli.cli_config_file import TomlProvider, configuration_option from samcli.cli.main import aws_creds_options, common_options, pass_context, print_cmdline_args from samcli.cli.types import ImageRepositoryType, ImageRepositoriesType +from samcli.commands.package.validations import package_option_validation +from samcli.commands._utils.iac_validations import iac_options_validation from samcli.commands._utils.options import ( capabilities_override_option, - guided_deploy_stack_name, metadata_override_option, notification_arns_override_option, parameter_override_option, @@ -19,6 +20,8 @@ template_click_option, signing_profiles_option, image_repositories_callback, + cdk_click_options, + project_type_click_option, ) from samcli.commands.deploy.utils import sanitize_parameter_overrides from samcli.lib.telemetry.metric import track_command @@ -26,6 +29,7 @@ from samcli.lib.utils import osutils from samcli.lib.bootstrap.bootstrap import manage_stack from samcli.lib.utils.version_checker import check_newer_version +from samcli.lib.iac.utils.helpers import inject_iac_plugin SHORT_HELP = "Deploy an AWS SAM application." @@ -49,6 +53,7 @@ help=HELP_TEXT, ) @configuration_option(provider=TomlProvider(section=CONFIG_SECTION)) +@project_type_click_option(include_build=True) @click.option( "--guided", "-g", @@ -61,10 +66,10 @@ @click.option( "--stack-name", required=False, - callback=guided_deploy_stack_name, help="The name of the AWS CloudFormation stack you're deploying to. " "If you specify an existing stack, the command updates the stack. " - "If you specify a new stack, the command creates it.", + "If you specify a new stack, the command creates it." + "If your project type is CDK, this option also specifies the stack to deploy.", ) @click.option( "--s3-bucket", @@ -165,6 +170,10 @@ @capabilities_override_option @aws_creds_options @common_options +@cdk_click_options +@inject_iac_plugin(with_build=True) +@iac_options_validation(require_stack=True) +@package_option_validation @image_repository_validation @pass_context @track_command @@ -196,6 +205,11 @@ def cli( resolve_s3, config_file, config_env, + project_type, + cdk_app, + cdk_context, + iac, + project, ): """ `sam deploy` command entry point @@ -228,6 +242,9 @@ def cli( resolve_s3, config_file, config_env, + project_type, + iac, + project, ) # pragma: no cover @@ -258,6 +275,9 @@ def do_cli( resolve_s3, config_file, config_env, + project_type, + iac, + project, ): """ Implementation of the ``cli`` method @@ -270,7 +290,6 @@ def do_cli( if guided: # Allow for a guided deploy to prompt and save those details. guided_context = GuidedContext( - template_file=template_file, stack_name=stack_name, s3_bucket=s3_bucket, image_repository=image_repository, @@ -285,6 +304,8 @@ def do_cli( config_section=CONFIG_SECTION, config_env=config_env, config_file=config_file, + iac=iac, + project=project, ) guided_context.run() elif resolve_s3 and bool(s3_bucket): @@ -313,6 +334,8 @@ def do_cli( region=guided_context.guided_region if guided else region, profile=profile, signing_profiles=guided_context.signing_profiles if guided else signing_profiles, + iac=iac, + project=project, ) as package_context: package_context.run() diff --git a/samcli/commands/deploy/deploy_context.py b/samcli/commands/deploy/deploy_context.py index 7af59e4711..e5a8566421 100644 --- a/samcli/commands/deploy/deploy_context.py +++ b/samcli/commands/deploy/deploy_context.py @@ -215,7 +215,9 @@ def deploy( template_dict = get_template_data(self.template_file) iac_stack.update(template_dict) stacks, _ = SamLocalStackProvider.get_stacks( - [iac_stack], parameter_overrides=sanitize_parameter_overrides(self.parameter_overrides) + [iac_stack], + parameter_overrides=sanitize_parameter_overrides(self.parameter_overrides), + normalize_resource_metadata=False, ) auth_required_per_resource = auth_per_resource(stacks) diff --git a/samcli/commands/deploy/guided_context.py b/samcli/commands/deploy/guided_context.py index 6c499ceb04..5a27f77b24 100644 --- a/samcli/commands/deploy/guided_context.py +++ b/samcli/commands/deploy/guided_context.py @@ -12,12 +12,6 @@ from click import confirm from samcli.commands._utils.options import _space_separated_list_func_type -from samcli.commands._utils.template import ( - get_template_parameters, - get_template_artifacts_format, - get_template_function_resource_ids, - get_template_data, -) from samcli.commands.deploy.code_signer_utils import ( signer_config_per_function, extract_profile_name_and_owner_from_existing, @@ -30,7 +24,6 @@ from samcli.commands.deploy.utils import sanitize_parameter_overrides from samcli.lib.config.samconfig import DEFAULT_ENV, DEFAULT_CONFIG_FILE_NAME from samcli.lib.bootstrap.bootstrap import manage_stack -from samcli.lib.iac.interface import Stack as IacStack from samcli.lib.package.ecr_utils import is_ecr_url from samcli.lib.package.image_utils import tag_translation, NonLocalImageException, NoImageFoundException from samcli.lib.providers.provider import Stack @@ -43,14 +36,15 @@ class GuidedContext: - def __init__( + def __init__( # pylint: disable=too-many-statements self, - template_file, stack_name, s3_bucket, image_repository, image_repositories, s3_prefix, + iac, + project, region=None, profile=None, confirm_changeset=None, @@ -62,7 +56,8 @@ def __init__( config_env=None, config_file=None, ): - self.template_file = template_file + self._iac = iac + self._project = project self.stack_name = stack_name self.s3_bucket = s3_bucket self.image_repository = image_repository @@ -91,6 +86,7 @@ def __init__( self.end_bold = "\033[0m" self.color = Colored() self.function_provider = None + self._get_iac_stack() @property def guided_capabilities(self): @@ -100,17 +96,41 @@ def guided_capabilities(self): def guided_parameter_overrides(self): return self._parameter_overrides + def _get_iac_stack(self): + """ + get iac_stack from project based on stack_name + """ + stack = None + if self.stack_name is not None: + stack = self._project.find_stack_by_name(self.stack_name) + if stack is None: + # there is not stack with provided stack name + raise GuidedDeployFailedError( + f"There is no stack with name '{self.stack_name}'. " + "If you have specified --stack-name, specify the correct stack name " + "or remove --stack-name to use default." + ) + + # NOTE: stack can be None because of one of the two reasons: + # 1) self.stack_name is None + # 2) self.stack_name is not None, but there is not stack with that name + # Either case, we select the first stack in the project by default, and update self.stack_name + if stack is None: + LOG.debug("Using the first stack in the project") + stack = self._project.stacks[0] + LOG.debug("name of first stack: '%s'", stack.name) + + self._iac_stack = stack + self.template_file = stack.origin_dir + self.stack_name = self.stack_name or stack.name or "sam-app" + # pylint: disable=too-many-statements - def guided_prompts(self, parameter_override_keys): + def guided_prompts(self): """ Start an interactive cli prompt to collection information for deployment - Parameters - ---------- - parameter_override_keys - The keys of parameters to override, for each key, customers will be asked to provide a value """ - default_stack_name = self.stack_name or "sam-app" + default_stack_name = self.stack_name default_region = self.region or get_session().get_config_variable("region") or "us-east-1" default_capabilities = self.capabilities[0] or ("CAPABILITY_IAM",) default_config_env = self.config_env or DEFAULT_ENV @@ -129,14 +149,12 @@ def guided_prompts(self, parameter_override_keys): f"\t{self.start_bold}Stack Name{self.end_bold}", default=default_stack_name, type=click.STRING ) region = prompt(f"\t{self.start_bold}AWS Region{self.end_bold}", default=default_region, type=click.STRING) + parameter_override_keys = self._iac_stack.get_overrideable_parameters() input_parameter_overrides = self.prompt_parameters( parameter_override_keys, self.parameter_overrides_from_cmdline, self.start_bold, self.end_bold ) - iac_stack = IacStack() - template_dict = get_template_data(self.template_file) - iac_stack.update(template_dict) stacks, _ = SamLocalStackProvider.get_stacks( - [iac_stack], parameter_overrides=sanitize_parameter_overrides(input_parameter_overrides) + [self._iac_stack], parameter_overrides=sanitize_parameter_overrides(input_parameter_overrides) ) image_repositories = self.prompt_image_repository(stacks) @@ -306,10 +324,11 @@ def prompt_image_repository(self, stacks: List[Stack]): A dictionary contains image function logical ID as key, image repository as value. """ image_repositories = {} - artifacts_format = get_template_artifacts_format(template_file=self.template_file) - if IMAGE in artifacts_format: + if self._iac_stack.has_assets_of_package_type(IMAGE): self.function_provider = SamFunctionProvider(stacks, ignore_code_extraction_warnings=True) - function_resources = get_template_function_resource_ids(template_file=self.template_file, artifact=IMAGE) + function_resources = [ + resource.item_id for resource in self._iac_stack.find_function_resources_of_package_type(IMAGE) + ] for resource_id in function_resources: image_repositories[resource_id] = prompt( f"\t{self.start_bold}Image Repository for {resource_id}{self.end_bold}", @@ -338,18 +357,12 @@ def prompt_image_repository(self, stacks: List[Stack]): def run(self): - try: - _parameter_override_keys = get_template_parameters(template_file=self.template_file) - except ValueError as ex: - LOG.debug("Failed to parse SAM template", exc_info=ex) - raise GuidedDeployFailedError(str(ex)) from ex - guided_config = GuidedConfig(template_file=self.template_file, section=self.config_section) guided_config.read_config_showcase( self.config_file or DEFAULT_CONFIG_FILE_NAME, ) - self.guided_prompts(_parameter_override_keys) + self.guided_prompts() if self.save_to_config: guided_config.save_config( diff --git a/samcli/commands/local/invoke/cli.py b/samcli/commands/local/invoke/cli.py index f5ca6a74f1..7ae8364a38 100644 --- a/samcli/commands/local/invoke/cli.py +++ b/samcli/commands/local/invoke/cli.py @@ -6,6 +6,7 @@ import click from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options, print_cmdline_args +from samcli.commands._utils.iac_validations import iac_options_validation from samcli.commands._utils.options import project_type_click_option, cdk_click_options from samcli.commands.local.cli_common.options import invoke_common_options, local_common_options from samcli.commands.local.lib.exceptions import InvalidIntermediateImageError @@ -45,16 +46,17 @@ "is not specified, no event is assumed. Pass in the value '-' to input JSON via stdin", ) @click.option("--no-event", is_flag=True, default=True, help="DEPRECATED: By default no event is assumed.", hidden=True) -@project_type_click_option +@project_type_click_option(include_build=True) @invoke_common_options @local_common_options @cli_framework_options @aws_creds_options @click.argument("function_logical_id", required=False) -@pass_context @cdk_click_options -@track_command # pylint: disable=R0914 @inject_iac_plugin(with_build=True) +@iac_options_validation(require_stack=False) +@pass_context +@track_command # pylint: disable=R0914 @check_newer_version @print_cmdline_args def cli( diff --git a/samcli/commands/local/start_api/cli.py b/samcli/commands/local/start_api/cli.py index 98fcea77c9..f1947038b3 100644 --- a/samcli/commands/local/start_api/cli.py +++ b/samcli/commands/local/start_api/cli.py @@ -6,6 +6,7 @@ import click from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options, print_cmdline_args +from samcli.commands._utils.iac_validations import iac_options_validation from samcli.commands._utils.options import project_type_click_option, cdk_click_options from samcli.commands.local.cli_common.options import ( invoke_common_options, @@ -49,16 +50,17 @@ default="public", help="Any static assets (e.g. CSS/Javascript/HTML) files located in this directory " "will be presented at /", ) -@project_type_click_option +@project_type_click_option(include_build=True) @invoke_common_options @warm_containers_common_options @local_common_options @cli_framework_options @aws_creds_options # pylint: disable=R0914 -@pass_context @cdk_click_options -@track_command @inject_iac_plugin(with_build=True) +@iac_options_validation(require_stack=False) +@pass_context +@track_command @check_newer_version @print_cmdline_args def cli( diff --git a/samcli/commands/local/start_lambda/cli.py b/samcli/commands/local/start_lambda/cli.py index 946b7ca276..6ff7af27fd 100644 --- a/samcli/commands/local/start_lambda/cli.py +++ b/samcli/commands/local/start_lambda/cli.py @@ -6,6 +6,7 @@ import click from samcli.cli.main import pass_context, common_options as cli_framework_options, aws_creds_options, print_cmdline_args +from samcli.commands._utils.iac_validations import iac_options_validation from samcli.commands._utils.options import project_type_click_option, cdk_click_options from samcli.commands.local.cli_common.options import ( invoke_common_options, @@ -62,16 +63,17 @@ ) @configuration_option(provider=TomlProvider(section="parameters")) @service_common_options(3001) -@project_type_click_option +@project_type_click_option(include_build=True) @invoke_common_options @warm_containers_common_options @local_common_options @cli_framework_options @aws_creds_options -@pass_context @cdk_click_options -@track_command @inject_iac_plugin(with_build=True) +@iac_options_validation(require_stack=False) +@pass_context +@track_command @check_newer_version @print_cmdline_args def cli( diff --git a/samcli/commands/package/command.py b/samcli/commands/package/command.py index cab68b6d88..33f242648c 100644 --- a/samcli/commands/package/command.py +++ b/samcli/commands/package/command.py @@ -1,28 +1,30 @@ """ CLI command for "package" command """ -from functools import partial import click from samcli.cli.cli_config_file import configuration_option, TomlProvider from samcli.cli.main import pass_context, common_options, aws_creds_options, print_cmdline_args from samcli.cli.types import ImageRepositoryType, ImageRepositoriesType -from samcli.commands.package.exceptions import PackageResolveS3AndS3SetError, PackageResolveS3AndS3NotSetError from samcli.lib.cli_validation.image_repository_validation import image_repository_validation -from samcli.lib.utils.packagetype import ZIP, IMAGE +from samcli.commands.package.validations import package_option_validation +from samcli.commands._utils.iac_validations import iac_options_validation from samcli.commands._utils.options import ( - artifact_callback, - resolve_s3_callback, signing_profiles_option, image_repositories_callback, + cdk_click_options, + project_type_click_option, + metadata_override_option, + template_click_option, + no_progressbar_option, ) -from samcli.commands._utils.options import metadata_override_option, template_click_option, no_progressbar_option from samcli.commands._utils.resources import resources_generator from samcli.lib.bootstrap.bootstrap import manage_stack from samcli.lib.telemetry.metric import track_command, track_template_warnings from samcli.lib.utils.version_checker import check_newer_version from samcli.lib.warnings.sam_cli_warning import CodeDeployWarning, CodeDeployConditionWarning +from samcli.lib.iac.utils.helpers import inject_iac_plugin SHORT_HELP = "Package an AWS SAM application." @@ -53,16 +55,15 @@ def resources_and_properties_help_string(): @click.command("package", short_help=SHORT_HELP, help=HELP_TEXT, context_settings=dict(max_content_width=120)) @configuration_option(provider=TomlProvider(section="parameters")) +@project_type_click_option(include_build=True) @template_click_option(include_build=True) @click.option( "--s3-bucket", required=False, - callback=partial(artifact_callback, artifact=ZIP), help="The name of the S3 bucket where this command uploads the artifacts that are referenced in your template.", ) @click.option( "--image-repository", - callback=partial(artifact_callback, artifact=IMAGE), type=ImageRepositoryType(), required=False, help="ECR repo uri where this command uploads the image artifacts that are referenced in your template.", @@ -115,25 +116,24 @@ def resources_and_properties_help_string(): "--resolve-s3", required=False, is_flag=True, - callback=partial( - resolve_s3_callback, - artifact=ZIP, - exc_set=PackageResolveS3AndS3SetError, - exc_not_set=PackageResolveS3AndS3NotSetError, - ), help="Automatically resolve s3 bucket for non-guided deployments." "Do not use --s3-guided parameter with this option.", ) +@click.option("--stack-name", required=False, help="The stack name to package") @metadata_override_option @signing_profiles_option @no_progressbar_option @common_options @aws_creds_options +@cdk_click_options +@inject_iac_plugin(with_build=True) +@iac_options_validation(require_stack=True) +@package_option_validation @image_repository_validation @pass_context @track_command -@check_newer_version @track_template_warnings([CodeDeployWarning.__name__, CodeDeployConditionWarning.__name__]) +@check_newer_version @print_cmdline_args def cli( ctx, @@ -152,6 +152,12 @@ def cli( resolve_s3, config_file, config_env, + project_type, + cdk_app, + cdk_context, + iac, + project, + stack_name, ): """ `sam package` command entry point @@ -174,6 +180,10 @@ def cli( ctx.region, ctx.profile, resolve_s3, + project_type, + iac, + project, + stack_name, ) # pragma: no cover @@ -193,6 +203,10 @@ def do_cli( region, profile, resolve_s3, + project_type, + iac, + project, + stack_name, ): """ Implementation of the ``cli`` method @@ -221,5 +235,8 @@ def do_cli( region=region, profile=profile, signing_profiles=signing_profiles, + iac=iac, + project=project, + stack_name=stack_name, ) as package_context: package_context.run() diff --git a/samcli/commands/package/package_context.py b/samcli/commands/package/package_context.py index 0a26577333..1814168668 100644 --- a/samcli/commands/package/package_context.py +++ b/samcli/commands/package/package_context.py @@ -31,6 +31,7 @@ from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.package.uploaders import Uploaders from samcli.lib.utils.botoconfig import get_boto_config_with_user_agent +from samcli.lib.iac.interface import DictSection from samcli.yamlhelper import yaml_dump LOG = logging.getLogger(__name__) @@ -66,6 +67,9 @@ def __init__( metadata, region, profile, + iac, + project, + stack_name=None, on_deploy=False, signing_profiles=None, ): @@ -85,6 +89,14 @@ def __init__( self.on_deploy = on_deploy self.code_signer = None self.signing_profiles = signing_profiles + self._iac = iac + self._project = project + # during validation, stack_name is required if project contains more than one stack + # we can safely assume project contains at least one stack as of now + if stack_name is not None and len(project.stacks) > 1: + self._stack = project.find_stack_by_name(stack_name) + else: + self._stack = project.stacks[0] def __enter__(self): return self @@ -119,7 +131,7 @@ def run(self): self.code_signer = CodeSigner(code_signer_client, self.signing_profiles) try: - exported_str = self._export(self.template_file, self.use_json) + exported_str = self._export(self.use_json) self.write_output(self.output_template_file, exported_str) @@ -132,9 +144,12 @@ def run(self): except OSError as ex: raise PackageFailedError(template_file=self.template_file, ex=str(ex)) from ex - def _export(self, template_path, use_json): - template = Template(template_path, os.getcwd(), self.uploaders, self.code_signer) + def _export(self, use_json): + template = Template(self._stack, os.getcwd(), self.uploaders, self.code_signer, self._iac) exported_template = template.export() + self._iac.update_asset_params_default_values_after_packaging( + exported_template, exported_template.get("Parameters", DictSection()) + ) if use_json: exported_str = json.dumps(exported_template, indent=4, ensure_ascii=False) diff --git a/samcli/commands/package/utils.py b/samcli/commands/package/utils.py new file mode 100644 index 0000000000..5d1c143654 --- /dev/null +++ b/samcli/commands/package/utils.py @@ -0,0 +1,25 @@ +""" +Package command options validations util +""" +import click + + +def validate_and_get_project_stack(project, ctx): + """ + Util to validate the stack to be packaged + it checks if the project contains only one stack, + or the stack-name option should be provided + """ + guided = ctx.params.get("guided", False) or ctx.params.get("g", False) + if len(project.stacks) == 1: + stack = project.stacks[0] + else: + stack_name = ctx.params.get("stack_name") + if stack_name is None and not guided: + raise click.BadOptionUsage( + option_name="--stack-name", + ctx=ctx, + message="You must specify stack name via --stack-name as your project contains more than one " "stack.", + ) + stack = project.find_stack_by_name(stack_name) + return stack diff --git a/samcli/commands/package/validations.py b/samcli/commands/package/validations.py new file mode 100644 index 0000000000..18f14411af --- /dev/null +++ b/samcli/commands/package/validations.py @@ -0,0 +1,43 @@ +""" +Option validations for Package command +Use as decorator and place the decorator after: +1. all CLI options have been processed. +2. iac plugin has been injected +""" +import functools +import logging + +import click +from samcli.commands.package.exceptions import PackageResolveS3AndS3NotSetError, PackageResolveS3AndS3SetError +from samcli.commands.package.utils import validate_and_get_project_stack +from samcli.lib.utils.packagetype import ZIP + +LOG = logging.getLogger(__name__) + + +def package_option_validation(func): + """ + Wrapper validation function that will run after cli parameters have been loaded + and iac plugin has been injected. + """ + + @functools.wraps(func) + def wrapped(*args, **kwargs): + project = kwargs.get("project") + ctx = click.get_current_context() + stack = validate_and_get_project_stack(project, ctx) + + # NOTE(sriram-mv): Both params and default_map need to be checked, as the option can be either be + # passed in directly or through configuration file. + # If passed in through configuration file, default_map is loaded with those values. + resolve_s3_provided = ctx.params.get("resolve_s3", False) or ctx.default_map.get("resolve_s3", False) + s3_bucket_provided = ctx.params.get("s3_bucket", False) or ctx.default_map.get("s3_bucket", False) + either_required = stack.has_assets_of_package_type(ZIP) if stack is not None else False + if s3_bucket_provided and resolve_s3_provided: + raise PackageResolveS3AndS3SetError() + if either_required and not s3_bucket_provided and not resolve_s3_provided: + raise PackageResolveS3AndS3NotSetError() + + return func(*args, **kwargs) + + return wrapped diff --git a/samcli/lib/cli_validation/image_repository_validation.py b/samcli/lib/cli_validation/image_repository_validation.py index 329e855019..35436d73e0 100644 --- a/samcli/lib/cli_validation/image_repository_validation.py +++ b/samcli/lib/cli_validation/image_repository_validation.py @@ -2,12 +2,15 @@ Image Repository Option Validation. This is to be run last after all CLI options have been processed. """ +import logging import click from samcli.commands._utils.option_validator import Validator -from samcli.commands._utils.template import get_template_function_resource_ids, get_template_artifacts_format +from samcli.commands.package.utils import validate_and_get_project_stack from samcli.lib.utils.packagetype import IMAGE +LOG = logging.getLogger(__name__) + def image_repository_validation(func): """ @@ -25,19 +28,12 @@ def wrapped(*args, **kwargs): guided = ctx.params.get("guided", False) or ctx.params.get("g", False) image_repository = ctx.params.get("image_repository", False) image_repositories = ctx.params.get("image_repositories", False) or {} - template_file = ( - ctx.params.get("t", False) or ctx.params.get("template_file", False) or ctx.params.get("template", False) - ) + project = kwargs.get("project") + stack = validate_and_get_project_stack(project, ctx) # Check if `--image-repository` or `--image-repositories` are required by # looking for resources that have an IMAGE based packagetype. - - required = any( - [ - _template_artifact == IMAGE - for _template_artifact in get_template_artifacts_format(template_file=template_file) - ] - ) + required = stack.has_assets_of_package_type(IMAGE) if stack is not None else False validators = [ Validator( @@ -60,7 +56,8 @@ def wrapped(*args, **kwargs): Validator( validation_function=lambda: not guided and ( - set(image_repositories.keys()) != set(get_template_function_resource_ids(template_file, IMAGE)) + set(image_repositories.keys()) + != set(map(lambda r: r.item_id, stack.find_function_resources_of_package_type(IMAGE))) and image_repositories ), exception=click.BadOptionUsage( diff --git a/samcli/lib/iac/cdk/cloud_assembly.py b/samcli/lib/iac/cdk/cloud_assembly.py index 090b8ffa39..de4ec337a1 100644 --- a/samcli/lib/iac/cdk/cloud_assembly.py +++ b/samcli/lib/iac/cdk/cloud_assembly.py @@ -257,7 +257,7 @@ def template(self) -> Dict: asset = self.find_asset_by_id(asset_id) if asset is not None: metadata[ASSET_PATH_METADATA_KEY] = asset["path"] - metadata[ASSET_PROPERTY_METADATA_KEY] = "Code.ImageUri" + metadata[ASSET_PROPERTY_METADATA_KEY] = "Code" self._template = template_dict return template_dict diff --git a/samcli/lib/iac/cdk/plugin.py b/samcli/lib/iac/cdk/plugin.py index 53a0884509..4e80d4d5bc 100644 --- a/samcli/lib/iac/cdk/plugin.py +++ b/samcli/lib/iac/cdk/plugin.py @@ -52,10 +52,8 @@ from samcli.lib.iac.cdk.exceptions import ( CdkSynthError, CdkToolkitNotInstalledError, - CdkPluginError, InvalidCloudAssemblyError, ) -from samcli.cli.context import Context from samcli.lib.samlib.resource_metadata_normalizer import ( METADATA_KEY, ASSET_PATH_METADATA_KEY, @@ -78,8 +76,8 @@ class CdkPlugin(IacPlugin): CDK Plugin """ - def __init__(self, context: Context): - super().__init__(context=context) + def __init__(self, command_params: dict): + super().__init__(command_params=command_params) # create a temp dir to hold synthed/read Cloud Assembly # will remove the temp dir once the command exits self._source_dir = os.path.abspath(os.curdir) @@ -90,9 +88,9 @@ def get_project(self, lookup_paths: List[LookupPath]) -> Project: """ Read and parse template of that IaC Platform """ - cdk_app = self._context.command_params.get("cdk_app") + cdk_app = self._command_params.get("cdk_app") is_cloud_assembly_dir = bool(cdk_app) and os.path.isfile(os.path.join(cdk_app, MANIFEST_FILENAME)) - cdk_context = self._context.command_params.get("cdk_context") + cdk_context = self._command_params.get("cdk_context") cloud_assembly_dir = None missing_files: List = [] for lookup_path in lookup_paths: @@ -129,15 +127,46 @@ def write_project(self, project: Project, build_dir: str) -> None: for stack in project.stacks: _write_stack(stack, self._cloud_assembly_dir, build_dir) - # p = subprocess.run(["tree", build_dir], capture_output=True) - # LOG.info(p) + def should_update_property_after_package(self, asset: Asset) -> bool: + if isinstance(asset, S3Asset): + # S3 Asset is binded with Asset Parameter. Thus, property should not be updated. + return False + return True + + def update_asset_params_default_values_after_packaging(self, stack: Stack, parameters: DictSection) -> None: + """ + Populate default values for asset parameters + """ + resources = stack.get("Resources", DictSection()) + for resource in resources.values(): + # undo normalize resource metadata + # update asset param default values + if resource.assets and resource.assets[0]: + asset = resource.assets[0] + if isinstance(asset, S3Asset) and "assetParameters" in asset.extra_details: + _update_asset_params_default_values(asset, parameters) + + # recursively do the same on nested stack + if resource.nested_stack: + self.update_asset_params_default_values_after_packaging(resource.nested_stack, parameters) + resource.assets = [] + resource.nested_stack = None + + def update_resource_after_packaging(self, resource: Resource) -> None: + """ + Update resource property to reference asset parameters + """ + if resource.assets and resource.assets[0]: + asset = resource.assets[0] + if isinstance(asset, S3Asset) and "assetParameters" in asset.extra_details: + _undo_normalize_resource_metadata(resource) def _cdk_synth(self, app: Optional[str] = None, context: Optional[List] = None) -> str: """ Run cdk synth to get the cloud assembly """ context = context or [] - cdk_executable = self._cdk_executable_path + cdk_executable = _get_cdk_executable_path() LOG.debug("CDK Toolkit found at %s", cdk_executable) synth_command = [ cdk_executable, @@ -174,31 +203,6 @@ def _cdk_synth(self, app: Optional[str] = None, context: Optional[List] = None) LOG.debug("Cloud assembly synthed at %s", self._cloud_assembly_dir) return self._cloud_assembly_dir - @property - def _cdk_executable_path(self) -> str: - """ - Order to look up locally installed CDK Toolkit - 1. ./node_modules/aws-cdk/bin/cdk (for mac & linux only) - 2. cdk - """ - if platform.system().lower() == "windows": - cdk_executables = ["cdk"] - else: - cdk_executables = [ - "./node_modules/aws-cdk/bin/cdk", - "cdk", - ] - - for executable in cdk_executables: - # check if exists and is executable - full_executable = shutil.which(executable) - if full_executable: - return full_executable - raise CdkToolkitNotInstalledError( - "CDK Toolkit is not found or not installed. Please run `npm i -g aws-cdk@latest` to install the latest CDK " - "Toolkit." - ) - def _get_project_from_cloud_assembly(self, cloud_assembly_dir: str) -> Project: """ create a cdk project from cloud_assembly @@ -215,6 +219,12 @@ def _build_stack( Extract stack from given CloudAssemblyStack """ assets = _collect_assets(ca_stack) + LOG.debug("Found assets: %s", str(assets)) + asset_parameters = { + asset_param + for asset in assets.values() + for asset_param in asset.extra_details.get("assetParameters", {}).values() + } sections = {} for section_key, section_dict in ca_stack.template.items(): if section_key == "Resources": @@ -227,6 +237,8 @@ def _build_stack( key=logical_id, body=param_dict, ) + if logical_id in asset_parameters: + param.added_by_iac = True section[logical_id] = param elif isinstance(section_dict, Mapping): section = DictSection(section_key) @@ -287,7 +299,12 @@ def _build_resources_section( # and keep an original copy for writing the project to template(s) original_body = copy.deepcopy(resource_dict) resource.extra_details[RESOURCE_EXTRA_DETAILS_ORIGINAL_BODY_KEY] = original_body - ResourceMetadataNormalizer.replace_property(asset_property, asset_path, resource, logical_id) + if ASSET_LOCAL_IMAGE_METADATA_KEY in metadata: + ResourceMetadataNormalizer.replace_property( + asset_property, metadata[ASSET_LOCAL_IMAGE_METADATA_KEY], resource, logical_id + ) + else: + ResourceMetadataNormalizer.replace_property(asset_property, asset_path, resource, logical_id) if resource_type in NESTED_STACKS_RESOURCES: # hook up and extract nested stacks @@ -305,15 +322,6 @@ def _build_resources_section( section[logical_id] = resource -def _get_app_executable_path_from_config() -> str: - with open(CDK_CONFIG_FILENAME, "r") as f: - app_json = json.loads(f.read()) - app: str = app_json.get("app") - if app is None: - raise CdkPluginError(f"'app' is missing in {CDK_CONFIG_FILENAME}") - return app - - def _collect_assets( ca_stack: Union[CloudAssemblyStack, CloudAssemblyNestedStack] ) -> Dict[str, Union[S3Asset, ImageAsset]]: @@ -322,7 +330,16 @@ def _collect_assets( if ca_asset["path"] not in assets: if ca_asset["packaging"] in [ZIP_ASSET_PACKAGING, FILE_ASSET_PACKAGING]: path = os.path.normpath(os.path.join(ca_stack.directory, ca_asset["path"])) - assets[ca_asset["path"]] = S3Asset(asset_id=ca_asset["id"], source_path=path) + extra_details = { + "assetParameters": { + "s3BucketParameter": ca_asset["s3BucketParameter"], + "s3KeyParameter": ca_asset["s3KeyParameter"], + "artifactHashParameter": ca_asset["artifactHashParameter"], + } + } + assets[ca_asset["path"]] = S3Asset( + asset_id=ca_asset["id"], source_path=path, extra_details=extra_details + ) elif ca_asset["packaging"] == CONTAINER_IMAGE_ASSET_PACKAGING: path = os.path.normpath(os.path.join(ca_stack.directory, ca_asset["path"])) repository_name = ca_asset.get("repositoryName", None) @@ -349,9 +366,7 @@ def _write_stack(stack: Stack, cloud_assembly_dir: str, build_dir: str) -> None: resources = stack.get("Resources", {}) for _, resource in resources.items(): - if RESOURCE_EXTRA_DETAILS_ORIGINAL_BODY_KEY in resource.extra_details: - for key, val in resource.extra_details[RESOURCE_EXTRA_DETAILS_ORIGINAL_BODY_KEY].items(): - resource[key] = val + _undo_normalize_resource_metadata(resource) if resource.assets: asset = resource.assets[0] if isinstance(asset, ImageAsset) and asset.source_local_image is not None: @@ -367,10 +382,16 @@ def _write_stack(stack: Stack, cloud_assembly_dir: str, build_dir: str) -> None: STACK_EXTRA_DETAILS_TEMPLATE_FILENAME_KEY ] asset.updated_source_path = os.path.join(build_dir, nested_stack_file_name) - resource[METADATA_KEY][ASSET_PATH_METADATA_KEY] = updated_path + resource[METADATA_KEY][ASSET_PATH_METADATA_KEY] = asset.updated_source_path move_template(src_template_path, stack_build_location, stack, output_format=TemplateFormat.JSON) +def _undo_normalize_resource_metadata(resource: Resource) -> None: + if RESOURCE_EXTRA_DETAILS_ORIGINAL_BODY_KEY in resource.extra_details: + for key, val in resource.extra_details[RESOURCE_EXTRA_DETAILS_ORIGINAL_BODY_KEY].items(): + resource[key] = val + + def _collect_stack_assets(stack: Stack) -> Dict[str, Asset]: collected_assets: Dict[str, Asset] = {} sections: Dict = stack.sections or {} @@ -451,3 +472,44 @@ def _collect_project_assets(project): assets[stack.name] = _collect_stack_assets(stack) root_stack_names.append(stack.name) return assets, root_stack_names + + +def _update_asset_params_default_values(asset: S3Asset, parameters: DictSection) -> None: + s3_bucket_param_key = asset.extra_details["assetParameters"].get("s3BucketParameter") + s3_bucket_param_val = asset.bucket_name + if s3_bucket_param_key is not None and s3_bucket_param_val is not None and s3_bucket_param_key in parameters: + parameters[s3_bucket_param_key]["Default"] = s3_bucket_param_val + s3_key_param_key = asset.extra_details["assetParameters"].get("s3KeyParameter") + s3_key_val = asset.object_key + s3_version_val = asset.object_version or "" + if s3_key_param_key is not None and s3_key_val is not None and s3_key_param_key in parameters: + parameters[s3_key_param_key]["Default"] = s3_key_val + "||" + s3_version_val + artifact_hash_key = asset.extra_details["assetParameters"].get("artifactHashParameter") + artifact_hash_val = asset.asset_id + if artifact_hash_key is not None and artifact_hash_val is not None and artifact_hash_key in parameters: + parameters[artifact_hash_key]["Default"] = artifact_hash_val + + +def _get_cdk_executable_path() -> str: + """ + Order to look up locally installed CDK Toolkit + 1. ./node_modules/aws-cdk/bin/cdk (for mac & linux only) + 2. cdk + """ + if platform.system().lower() == "windows": + cdk_executables = ["cdk"] + else: + cdk_executables = [ + "./node_modules/aws-cdk/bin/cdk", + "cdk", + ] + + for executable in cdk_executables: + # check if exists and is executable + full_executable = shutil.which(executable) + if full_executable: + return full_executable + raise CdkToolkitNotInstalledError( + "CDK Toolkit is not found or not installed. Please run `npm i -g aws-cdk@latest` to install the latest CDK " + "Toolkit." + ) diff --git a/samcli/lib/iac/cfn_iac.py b/samcli/lib/iac/cfn_iac.py index 39f626a2e9..39ee87b73d 100644 --- a/samcli/lib/iac/cfn_iac.py +++ b/samcli/lib/iac/cfn_iac.py @@ -2,26 +2,37 @@ Cloud Formation IaC plugin implementation """ import os +import logging from typing import List, Optional from urllib.parse import unquote, urlparse import jmespath -from samcli.cli.context import Context -from samcli.commands._utils.resources import RESOURCES_WITH_LOCAL_PATHS -from samcli.lib.iac.interface import IacPlugin, Project, LookupPath, Stack, DictSection, S3Asset, Asset +from samcli.commands._utils.resources import ( + RESOURCES_WITH_IMAGE_COMPONENT, + RESOURCES_WITH_LOCAL_PATHS, + NESTED_STACKS_RESOURCES, +) +from samcli.lib.iac.interface import ( + IacPlugin, + ImageAsset, + Project, + LookupPath, + Stack, + DictSection, + S3Asset, + Asset, + Resource, +) from samcli.commands._utils.template import get_template_data, move_template from samcli.lib.providers.sam_stack_provider import SamLocalStackProvider, is_local_path, get_local_path +LOG = logging.getLogger(__name__) + PARENT_STACK_TEMPLATE_PATH_KEY = "parent_stack_template_path" TEMPLATE_PATH_KEY = "template_path" TEMPLATE_BUILD_PATH_KEY = "template_build_path" -NESTED_STACKS_RESOURCES = { - SamLocalStackProvider.SERVERLESS_APPLICATION: "Location", - SamLocalStackProvider.CLOUDFORMATION_STACK: "TemplateURL", -} - BASE_DIR_RESOURCES = [ SamLocalStackProvider.SERVERLESS_FUNCTION, SamLocalStackProvider.LAMBDA_FUNCTION, @@ -31,10 +42,10 @@ class CfnIacPlugin(IacPlugin): - def __init__(self, context: Context): - self._template_file = context.command_params["template_file"] - self._base_dir = context.command_params.get("base_dir", None) - super().__init__(context) + def __init__(self, command_params: dict): + self._template_file = command_params["template_file"] + self._base_dir = command_params.get("base_dir", None) + super().__init__(command_params) def get_project(self, lookup_paths: List[LookupPath]) -> Project: stacks = [self._build_stack(self._template_file)] @@ -75,6 +86,15 @@ def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = asset_path = get_local_path(asset_path, reference_path) asset = S3Asset(source_path=asset_path, source_property=path_prop_name) resource_assets.append(asset) + stack.assets.append(asset) + + if resource_type in RESOURCES_WITH_IMAGE_COMPONENT: + for path_prop_name in RESOURCES_WITH_IMAGE_COMPONENT[resource_type]: + asset_path = jmespath.search(path_prop_name, properties) + if asset_path: + asset = ImageAsset(source_local_image=asset_path, source_property=path_prop_name) + resource_assets.append(asset) + stack.assets.append(asset) resource.assets = resource_assets @@ -105,6 +125,15 @@ def write_project(self, project: Project, build_dir: str) -> None: for stack in project.stacks: _write_stack(stack, build_dir) + def should_update_property_after_package(self, asset: Asset) -> bool: + return True + + def update_resource_after_packaging(self, resource: Resource) -> None: + pass + + def update_asset_params_default_values_after_packaging(self, stack: Stack, parameters: DictSection) -> None: + pass + def _write_stack(stack: Stack, build_dir: str): stack_id = stack.stack_id or "" diff --git a/samcli/lib/iac/interface.py b/samcli/lib/iac/interface.py index 47b80191b9..197a3b084b 100644 --- a/samcli/lib/iac/interface.py +++ b/samcli/lib/iac/interface.py @@ -3,15 +3,25 @@ """ import abc +import logging from collections import OrderedDict from collections.abc import MutableMapping, Mapping from copy import deepcopy from enum import Enum - from typing import List, Any, Dict, Iterator, Optional from uuid import uuid4 -from samcli.cli.context import Context +from samcli.commands._utils.resources import ( + AWS_LAMBDA_FUNCTION, + AWS_SERVERLESS_FUNCTION, + RESOURCES_WITH_IMAGE_COMPONENT, + RESOURCES_WITH_LOCAL_PATHS, + NESTED_STACKS_RESOURCES, +) +from samcli.lib.utils.packagetype import IMAGE, ZIP + + +LOG = logging.getLogger(__name__) class Environment: @@ -64,6 +74,7 @@ def __init__( asset_id: Optional[str] = None, destinations: Optional[List[Destination]] = None, source_property: Optional[str] = None, + extra_details: Optional[Dict[str, Any]] = None, ): if asset_id is None: asset_id = str(uuid4()) @@ -72,6 +83,8 @@ def __init__( destinations = [] self._destinations = destinations self._source_property = source_property + extra_details = extra_details or {} + self._extra_details = extra_details @property def asset_id(self) -> str: @@ -97,6 +110,14 @@ def source_property(self) -> Optional[str]: def source_property(self, source_property: str) -> None: self._source_property = source_property + @property + def extra_details(self) -> Dict[str, Any]: + return self._extra_details + + @extra_details.setter + def extra_details(self, extra_details: Dict[str, Any]) -> None: + self._extra_details = extra_details + class S3Asset(Asset): """ @@ -114,13 +135,14 @@ def __init__( updated_source_path: Optional[str] = None, destinations: Optional[List[Destination]] = None, source_property: Optional[str] = None, + extra_details: Optional[Dict[str, Any]] = None, ): self._bucket_name = bucket_name self._object_key = object_key self._object_version = object_version self._source_path = source_path self._updated_source_path = updated_source_path - super().__init__(asset_id, destinations, source_property) + super().__init__(asset_id, destinations, source_property, extra_details) @property def bucket_name(self) -> Optional[str]: @@ -181,6 +203,7 @@ def __init__( destinations: Optional[List[Destination]] = None, source_property: Optional[str] = None, target: Optional[str] = None, + extra_details: Optional[Dict[str, Any]] = None, ): """ image uri = /repository_name:image_tag @@ -194,7 +217,7 @@ def __init__( self._docker_file_name = docker_file_name self._build_args = build_args self._target = target - super().__init__(asset_id, destinations, source_property) + super().__init__(asset_id, destinations, source_property, extra_details) @property def repository_name(self) -> Optional[str]: @@ -320,7 +343,7 @@ def __init__( extra_details: Optional[Dict[str, Any]] = None, ): super().__init__(key, item_id) - self._body = body + self._body = body or {} if assets is None: assets = [] self._assets = assets @@ -472,6 +495,24 @@ def nested_stack(self) -> Optional["Stack"]: def nested_stack(self, nested_stack: "Stack") -> None: self._nested_stack = nested_stack + def is_packageable(self): + """ + return if the resource is packageable + NOTE: we probably want to include a condition to check if the resource is binded to a local asset + But in samcli.lib.package.utils.upload_local_artifacts, we handle a case where local_path is None, + we would build the parent_dir. This seems to only apply for CFN/SAM project, we can consider to move + that logic to CfnIacPlugin. For now, just keep it as is. + """ + if "InlineCode" in self.get("Properties", {}): + return False + resource_type = self.get("Type", None) + packageable_resources = [ + NESTED_STACKS_RESOURCES, + RESOURCES_WITH_LOCAL_PATHS, + RESOURCES_WITH_IMAGE_COMPONENT, + ] + return any(resource_type in p for p in packageable_resources) + class Parameter(DictSectionItem): """ @@ -600,6 +641,34 @@ def extra_details(self) -> Dict[str, Any]: def extra_details(self, extra_details: Dict[str, Any]) -> None: self._extra_details = extra_details + def has_assets_of_package_type(self, package_type: str) -> bool: + package_type_to_asset_cls_map = { + ZIP: S3Asset, + IMAGE: ImageAsset, + } + return any(isinstance(asset, package_type_to_asset_cls_map[package_type]) for asset in self.assets) + + def find_function_resources_of_package_type(self, package_type: str) -> List[Resource]: + package_type_to_asset_cls_map = { + ZIP: S3Asset, + IMAGE: ImageAsset, + } + _function_resources = [] + for _, resource in self.get("Resources", DictSection()).items(): + if ( + resource.get("Type", "") in [AWS_SERVERLESS_FUNCTION, AWS_LAMBDA_FUNCTION] + and resource.assets + and isinstance(resource.assets[0], package_type_to_asset_cls_map[package_type]) + ): + _function_resources.append(resource) + return _function_resources + + def as_dict(self): + """ + return the stack as a dict for JSON serialization + """ + return _make_dict(self) + def __setitem__(self, k: str, v: Any) -> None: if isinstance(v, dict): section = DictSection(section_name=k) @@ -658,6 +727,12 @@ def extra_details(self) -> Optional[Dict[str, Any]]: def extra_details(self, extra_details: Dict[str, Any]) -> None: self._extra_details = extra_details + def find_stack_by_name(self, name: str): + for stack in self.stacks: + if stack.name == name: + return stack + return None + class LookupPathType(Enum): SOURCE = "Source" @@ -697,8 +772,8 @@ class IacPlugin(metaclass=abc.ABCMeta): We only require two methods here - get_project and write_project """ - def __init__(self, context: Context): - self._context = context + def __init__(self, command_params: dict): + self._command_params = command_params @abc.abstractmethod def get_project(self, lookup_paths: List[LookupPath]) -> Project: @@ -713,3 +788,30 @@ def write_project(self, project: Project, build_dir: str) -> None: Write project to a template (or a set of templates), move the template(s) to build_path """ + + @abc.abstractmethod + def should_update_property_after_package(self, asset: Asset) -> bool: + """ + return if resource property should be updated after packaging + """ + + @abc.abstractmethod + def update_resource_after_packaging(self, resource: Resource) -> None: + """ + Update resource after packaging, e.g. uploaded to S3 or ECR + """ + + @abc.abstractmethod + def update_asset_params_default_values_after_packaging(self, stack: Stack, parameters: DictSection) -> None: + """ + Populate default values for asset parameters + """ + + +def _make_dict(obj): + if not isinstance(obj, MutableMapping): + return obj + to_return = dict() + for key, val in obj.items(): + to_return[key] = _make_dict(val) + return to_return diff --git a/samcli/lib/iac/utils/helpers.py b/samcli/lib/iac/utils/helpers.py index dc9e6cef8e..c929524acc 100644 --- a/samcli/lib/iac/utils/helpers.py +++ b/samcli/lib/iac/utils/helpers.py @@ -1,38 +1,58 @@ """ Provide Helper methods and decorators to process IAC Plugins """ +import logging import pathlib -from samcli.cli.context import Context +import click + from samcli.lib.iac.cfn_iac import CfnIacPlugin from samcli.lib.iac.cdk.plugin import CdkPlugin from samcli.lib.iac.interface import ProjectTypes, LookupPath, LookupPathType +LOG = logging.getLogger(__name__) + + +def get_iac_plugin(project_type, command_params, with_build): + LOG.debug("IAC Plugin getting project...") + iac_plugins = { + ProjectTypes.CFN.value: CfnIacPlugin, + ProjectTypes.CDK.value: CdkPlugin, + } + if project_type is None or project_type not in iac_plugins: + raise click.BadOptionUsage( + option_name="--project-type", + message=f"{project_type} is invalid project type option value, the value should be one" + f"of the following {[ptype.value for ptype in ProjectTypes]} ", + ) + iac_plugin = iac_plugins[project_type](command_params) + lookup_paths = [] + + if with_build: + from samcli.commands.build.command import DEFAULT_BUILD_DIR + + # is this correct? --build-dir is only used for "build" (for writing) + # but with_true is True for "local" commands only + build_dir = command_params.get("build_dir", DEFAULT_BUILD_DIR) + lookup_paths.append(LookupPath(build_dir, LookupPathType.BUILD)) + lookup_paths.append(LookupPath(str(pathlib.Path.cwd()), LookupPathType.SOURCE)) + project = iac_plugin.get_project(lookup_paths) + + return iac_plugin, project + + def inject_iac_plugin(with_build: bool): def inner(func): def wrapper(*args, **kwargs): project_type = kwargs.get("project_type", ProjectTypes.CFN.value) - iac_plugins = { - ProjectTypes.CFN.value: CfnIacPlugin, - ProjectTypes.CDK.value: CdkPlugin, - } - ctx = Context.get_current_context() - iac_plugin = iac_plugins[project_type](ctx) - lookup_paths = [] - if with_build: - from samcli.commands.build.command import DEFAULT_BUILD_DIR - - build_dir = kwargs.get("build_dir", DEFAULT_BUILD_DIR) - lookup_paths.append(LookupPath(build_dir, LookupPathType.BUILD)) - lookup_paths.append(LookupPath(str(pathlib.Path.cwd()), LookupPathType.SOURCE)) - project = iac_plugin.get_project(lookup_paths) + iac_plugin, project = get_iac_plugin(project_type, kwargs, with_build) kwargs["iac"] = iac_plugin kwargs["project"] = project - func(*args, **kwargs) + return func(*args, **kwargs) return wrapper diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index 8ef0652f47..d986ccf261 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -16,7 +16,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import os -from typing import Dict +import logging +from typing import Dict, Union from botocore.utils import set_value_from_jmespath @@ -36,13 +37,16 @@ ) from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.package.uploaders import Uploaders -from samcli.lib.package.utils import is_local_folder, make_abs_path, is_s3_url, is_local_file, mktempfile +from samcli.lib.package.utils import is_local_folder, mktempfile, is_s3_url, is_local_file, make_abs_path from samcli.lib.utils.packagetype import ZIP -from samcli.yamlhelper import yaml_parse, yaml_dump +from samcli.yamlhelper import yaml_dump +from samcli.lib.iac.interface import Stack as IacStack, IacPlugin # NOTE: sriram-mv, A cyclic dependency on `Template` needs to be broken. +LOG = logging.getLogger(__name__) + class CloudFormationStackResource(ResourceZip): """ @@ -53,15 +57,25 @@ class CloudFormationStackResource(ResourceZip): RESOURCE_TYPE = AWS_CLOUDFORMATION_STACK PROPERTY_NAME = RESOURCES_WITH_LOCAL_PATHS[RESOURCE_TYPE][0] - def do_export(self, resource_id, resource_dict, parent_dir): + # pylint: disable=fixme + # FIXME: add type annotation once MRO fixed in Iac interface + def export(self, resource, parent_dir): + if not resource.nested_stack: + return + super().export(resource, parent_dir) + + # FIXME: add type annotation once MRO fixed in Iac interface + def do_export(self, resource, parent_dir): """ If the nested stack template is valid, this method will export on the nested template, upload the exported template to S3 and set property to URL of the uploaded S3 template """ - template_path = resource_dict.get(self.PROPERTY_NAME, None) + resource_dict = resource.get("Properties", {}) + asset = resource.assets[0] + template_path = asset.source_path if ( template_path is None or is_s3_url(template_path) @@ -74,10 +88,12 @@ def do_export(self, resource_id, resource_dict, parent_dir): abs_template_path = make_abs_path(parent_dir, template_path) if not is_local_file(abs_template_path): raise exceptions.InvalidTemplateUrlParameterError( - property_name=self.PROPERTY_NAME, resource_id=resource_id, template_path=abs_template_path + property_name=self.PROPERTY_NAME, resource_id=resource.key, template_path=abs_template_path ) - exported_template_dict = Template(template_path, parent_dir, self.uploaders, self.code_signer).export() + exported_template_dict = Template( + resource.nested_stack, parent_dir, self.uploaders, self.code_signer, self.iac + ).export() exported_template_str = yaml_dump(exported_template_dict) @@ -89,8 +105,15 @@ def do_export(self, resource_id, resource_dict, parent_dir): # TemplateUrl property requires S3 URL to be in path-style format parts = S3Uploader.parse_s3_url(url, version_property="Version") - s3_path_url = self.uploader.to_path_style_s3_url(parts["Key"], parts.get("Version", None)) - set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, s3_path_url) + s3_path_url = self.uploader.to_path_style_s3_url(parts.get("Key"), parts.get("Version")) + + asset = resource.assets[0] + asset.bucket_name = parts.get("Bucket") + asset.object_key = parts.get("Key") + asset.object_version = parts.get("Version") + self.iac.update_resource_after_packaging(resource) + if self.iac.should_update_property_after_package: + set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, s3_path_url) class ServerlessApplicationResource(CloudFormationStackResource): @@ -108,7 +131,7 @@ class Template: Class to export a CloudFormation template """ - template_dict: Dict + template_dict: IacStack template_dir: str resources_to_export: frozenset metadata_to_export: frozenset @@ -117,10 +140,11 @@ class Template: def __init__( self, - template_path: str, + template_dict: IacStack, parent_dir: str, uploaders: Uploaders, code_signer: CodeSigner, + iac: IacPlugin, resources_to_export=frozenset( RESOURCES_EXPORT_LIST + [CloudFormationStackResource, ServerlessApplicationResource] ), @@ -132,27 +156,22 @@ def __init__( if not (is_local_folder(parent_dir) and os.path.isabs(parent_dir)): raise ValueError("parent_dir parameter must be " "an absolute path to a folder {0}".format(parent_dir)) - abs_template_path = make_abs_path(parent_dir, template_path) - template_dir = os.path.dirname(abs_template_path) - - with open(abs_template_path, "r") as handle: - template_str = handle.read() - - self.template_dict = yaml_parse(template_str) - self.template_dir = template_dir + self.template_dict = template_dict + self.template_dir = template_dict.origin_dir self.resources_to_export = resources_to_export self.metadata_to_export = metadata_to_export self.uploaders = uploaders self.code_signer = code_signer + self.iac = iac - def _export_global_artifacts(self, template_dict: Dict) -> Dict: + def _export_global_artifacts(self, template_dict: Union[IacStack, Dict]): """ Template params such as AWS::Include transforms are not specific to any resource type but contain artifacts that should be exported, here we iterate through the template dict and export params with a handler defined in GLOBAL_EXPORT_DICT """ - for key, val in template_dict.items(): + for key, val in template_dict.items(): # type: ignore if key in GLOBAL_EXPORT_DICT: template_dict[key] = GLOBAL_EXPORT_DICT[key]( val, self.uploaders.get(ResourceZip.EXPORT_DESTINATION), self.template_dir @@ -163,7 +182,6 @@ def _export_global_artifacts(self, template_dict: Dict) -> Dict: for item in val: if isinstance(item, dict): self._export_global_artifacts(item) - return template_dict def _export_metadata(self): """ @@ -203,7 +221,7 @@ def _apply_global_values(self): if code_uri_global is not None and resource_dict is not None: resource_dict["CodeUri"] = code_uri_global - def export(self) -> Dict: + def export(self) -> IacStack: """ Exports the local artifacts referenced by the given template to an export destination. @@ -217,9 +235,9 @@ def export(self) -> Dict: return self.template_dict self._apply_global_values() - self.template_dict = self._export_global_artifacts(self.template_dict) + self._export_global_artifacts(self.template_dict) - for resource_id, resource in self.template_dict["Resources"].items(): + for resource in self.template_dict["Resources"].values(): # type: ignore resource_type = resource.get("Type", None) resource_dict = resource.get("Properties", {}) @@ -230,7 +248,7 @@ def export(self) -> Dict: if resource_dict.get("PackageType", ZIP) != exporter_class.ARTIFACT_TYPE: continue # Export code resources - exporter = exporter_class(self.uploaders, self.code_signer) - exporter.export(resource_id, resource_dict, self.template_dir) + exporter = exporter_class(self.uploaders, self.code_signer, self.iac) + exporter.export(resource, self.template_dir) return self.template_dict diff --git a/samcli/lib/package/packageable_resources.py b/samcli/lib/package/packageable_resources.py index edb99074d8..4dbf331c4e 100644 --- a/samcli/lib/package/packageable_resources.py +++ b/samcli/lib/package/packageable_resources.py @@ -4,17 +4,16 @@ import logging import os import shutil -from typing import Optional, Union, Dict +from typing import Optional, Union -import jmespath from botocore.utils import set_value_from_jmespath from samcli.commands.package import exceptions +from samcli.lib.iac.interface import S3Asset, ImageAsset from samcli.lib.package.ecr_uploader import ECRUploader from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.package.uploaders import Destination, Uploaders from samcli.lib.package.utils import ( - resource_not_packageable, is_local_file, is_zip_file, copy_to_temp_dir, @@ -52,6 +51,7 @@ LOG = logging.getLogger(__name__) +# pylint: disable=fixme class Resource: RESOURCE_TYPE: Optional[str] = None PROPERTY_NAME: Optional[str] = None @@ -62,9 +62,10 @@ class Resource: EXPORT_DESTINATION: Destination ARTIFACT_TYPE: Optional[str] = None - def __init__(self, uploaders: Uploaders, code_signer): + def __init__(self, uploaders: Uploaders, code_signer, iac): self.uploaders = uploaders self.code_signer = code_signer + self.iac = iac @property def uploader(self) -> Union[S3Uploader, ECRUploader]: @@ -73,10 +74,12 @@ def uploader(self) -> Union[S3Uploader, ECRUploader]: """ return self.uploaders.get(self.EXPORT_DESTINATION) - def export(self, resource_id, resource_dict, parent_dir): - self.do_export(resource_id, resource_dict, parent_dir) + # FIXME: add type annotation once MRO fixed in Iac interface + def export(self, resource, parent_dir): + self.do_export(resource, parent_dir) - def do_export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def do_export(self, resource, parent_dir): pass @@ -94,42 +97,44 @@ class ResourceZip(Resource): ARTIFACT_TYPE = ZIP EXPORT_DESTINATION = Destination.S3 - def export(self, resource_id: str, resource_dict: Optional[Dict], parent_dir: str): + # FIXME: add type annotation once MRO fixed in Iac interface + def export(self, resource, parent_dir): + resource_id = resource.key + resource_dict = resource.get("Properties", {}) if resource_dict is None: return - if resource_not_packageable(resource_dict): + # With IaC, we consider a resource packageable if it has an asset binded + if not resource.is_packageable(): return - property_value = jmespath.search(self.PROPERTY_NAME, resource_dict) + # we can safely assume resource.assets contains at lease one asset + asset = resource.assets[0] - if not property_value and not self.PACKAGE_NULL_PROPERTY: - return - - if isinstance(property_value, dict): - LOG.debug("Property %s of %s resource is not a URL", self.PROPERTY_NAME, resource_id) + if not asset.source_path and not self.PACKAGE_NULL_PROPERTY: return # If property is a file but not a zip file, place file in temp # folder and send the temp folder to be zipped temp_dir = None - if is_local_file(property_value) and not is_zip_file(property_value) and self.FORCE_ZIP: - temp_dir = copy_to_temp_dir(property_value) - set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, temp_dir) + if is_local_file(asset.source_path) and not is_zip_file(asset.source_path) and self.FORCE_ZIP: + temp_dir = copy_to_temp_dir(asset.source_path) + asset.source_path = temp_dir try: - self.do_export(resource_id, resource_dict, parent_dir) + self.do_export(resource, parent_dir) except Exception as ex: LOG.debug("Unable to export", exc_info=ex) raise exceptions.ExportFailedError( - resource_id=resource_id, property_name=self.PROPERTY_NAME, property_value=property_value, ex=ex + resource_id=resource_id, property_name=self.PROPERTY_NAME, property_value=asset.source_path, ex=ex ) finally: if temp_dir: shutil.rmtree(temp_dir) - def do_export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def do_export(self, resource, parent_dir): """ Default export action is to upload artifacts and set the property to S3 URL of the uploaded object @@ -138,11 +143,18 @@ def do_export(self, resource_id, resource_dict, parent_dir): """ # code signer only accepts files which has '.zip' extension in it # so package artifact with '.zip' if it is required to be signed + resource_id = resource.key + resource_dict = resource.get("Properties", {}) + + if not (resource.assets and isinstance(resource.assets[0], S3Asset)): + return + + asset = resource.assets[0] should_sign_package = self.code_signer.should_sign_package(resource_id) artifact_extension = "zip" if should_sign_package else None uploaded_url = upload_local_artifacts( resource_id, - resource_dict, + asset, self.PROPERTY_NAME, parent_dir, self.uploader, @@ -152,7 +164,8 @@ def do_export(self, resource_id, resource_dict, parent_dir): uploaded_url = self.code_signer.sign_package( resource_id, uploaded_url, self.uploader.get_version_of_artifact(uploaded_url) ) - set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) + if self.iac.should_update_property_after_package: + set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) class ResourceImageDict(Resource): @@ -167,35 +180,48 @@ class ResourceImageDict(Resource): EXPORT_DESTINATION = Destination.ECR EXPORT_PROPERTY_CODE_KEY = "ImageUri" - def export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def export(self, resource, parent_dir): + resource_id = resource.key + resource_dict = resource.get("Properties", {}) if resource_dict is None: return - property_value = jmespath.search(self.PROPERTY_NAME, resource_dict) - - if isinstance(property_value, dict): - LOG.debug("Property %s of %s resource is not a URL or a local image", self.PROPERTY_NAME, resource_id) + # With IaC, we consider a resource packageable if it has an asset binded + if not resource.is_packageable(): return try: - self.do_export(resource_id, resource_dict, parent_dir) + self.do_export(resource, parent_dir) except Exception as ex: LOG.debug("Unable to export", exc_info=ex) raise exceptions.ExportFailedError( - resource_id=resource_id, property_name=self.PROPERTY_NAME, property_value=property_value, ex=ex + resource_id=resource_id, + property_name=self.PROPERTY_NAME, + property_value=resource.assets[0].source_local_image, + ex=ex, ) - def do_export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def do_export(self, resource, parent_dir): """ Default export action is to upload artifacts and set the property to dictionary where the key is EXPORT_PROPERTY_CODE_KEY and value is set to an uploaded URL. """ + resource_id = resource.key + resource_dict = resource.get("Properties", {}) + + if not (resource.assets and isinstance(resource.assets[0], ImageAsset)): + return + uploaded_url = upload_local_image_artifacts( - resource_id, resource_dict, self.PROPERTY_NAME, parent_dir, self.uploader + resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader ) - set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, {self.EXPORT_PROPERTY_CODE_KEY: uploaded_url}) + self.iac.update_resource_after_packaging(resource) + if self.iac.should_update_property_after_package: + set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, {self.EXPORT_PROPERTY_CODE_KEY: uploaded_url}) class ResourceImage(Resource): @@ -209,34 +235,43 @@ class ResourceImage(Resource): ARTIFACT_TYPE: Optional[str] = IMAGE EXPORT_DESTINATION = Destination.ECR - def export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def export(self, resource, parent_dir): + resource_id = resource.key + resource_dict = resource.get("Properties", {}) if resource_dict is None: return - property_value = jmespath.search(self.PROPERTY_NAME, resource_dict) - - if isinstance(property_value, dict): - LOG.debug("Property %s of %s resource is not a URL or a local image", self.PROPERTY_NAME, resource_id) + # With IaC, we consider a resource packageable if it has an asset binded + if not resource.is_packageable(): return try: - self.do_export(resource_id, resource_dict, parent_dir) + self.do_export(resource, parent_dir) except Exception as ex: LOG.debug("Unable to export", exc_info=ex) raise exceptions.ExportFailedError( - resource_id=resource_id, property_name=self.PROPERTY_NAME, property_value=property_value, ex=ex + resource_id=resource_id, + property_name=self.PROPERTY_NAME, + property_value=resource.assets[0].source_local_image, + ex=ex, ) - def do_export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def do_export(self, resource, parent_dir): """ Default export action is to upload artifacts and set the property to URL of the uploaded object """ + resource_id = resource.key + resource_dict = resource.get("Properties", {}) uploaded_url = upload_local_image_artifacts( - resource_id, resource_dict, self.PROPERTY_NAME, parent_dir, self.uploader + resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader ) - set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) + self.iac.update_resource_after_packaging(resource) + if self.iac.should_update_property_after_package: + set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) class ResourceWithS3UrlDict(ResourceZip): @@ -251,15 +286,17 @@ class ResourceWithS3UrlDict(ResourceZip): ARTIFACT_TYPE = ZIP EXPORT_DESTINATION = Destination.S3 - def do_export(self, resource_id, resource_dict, parent_dir): + # FIXME: add type annotation once MRO fixed in Iac interface + def do_export(self, resource, parent_dir): """ Upload to S3 and set property to an dict representing the S3 url of the uploaded object """ + resource_id = resource.key + resource_dict = resource.get("Properties", {}) + asset = resource.assets[0] - artifact_s3_url = upload_local_artifacts( - resource_id, resource_dict, self.PROPERTY_NAME, parent_dir, self.uploader - ) + artifact_s3_url = upload_local_artifacts(resource_id, asset, self.PROPERTY_NAME, parent_dir, self.uploader) parsed_url = S3Uploader.parse_s3_url( artifact_s3_url, @@ -267,7 +304,12 @@ def do_export(self, resource_id, resource_dict, parent_dir): object_key_property=self.OBJECT_KEY_PROPERTY, version_property=self.VERSION_PROPERTY, ) - set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, parsed_url) + asset.bucket_name = parsed_url[self.BUCKET_NAME_PROPERTY] + asset.object_key = parsed_url[self.OBJECT_KEY_PROPERTY] + asset.object_version = parsed_url.get(self.VERSION_PROPERTY, None) + self.iac.update_resource_after_packaging(resource) + if self.iac.should_update_property_after_package: + set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, parsed_url) class ServerlessFunctionResource(ResourceZip): diff --git a/samcli/lib/package/utils.py b/samcli/lib/package/utils.py index 54bb81ba97..a0f6961bf8 100644 --- a/samcli/lib/package/utils.py +++ b/samcli/lib/package/utils.py @@ -10,7 +10,7 @@ import zipfile import contextlib from contextlib import contextmanager -from typing import Dict, Optional, cast +from typing import Optional import jmespath @@ -19,6 +19,7 @@ from samcli.lib.package.ecr_utils import is_ecr_url from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.utils.hash import dir_checksum +from samcli.lib.iac.interface import S3Asset as IacS3Asset LOG = logging.getLogger(__name__) @@ -53,7 +54,7 @@ def is_zip_file(path): return is_path_value_valid(path) and zipfile.is_zipfile(path) -def upload_local_image_artifacts(resource_id, resource_dict, property_name, parent_dir, uploader): +def upload_local_image_artifacts(resource_id, asset, property_name, parent_dir, uploader): """ Upload local artifacts referenced by the property at given resource and return ECR URL of the uploaded object. It is the responsibility of callers @@ -62,7 +63,7 @@ def upload_local_image_artifacts(resource_id, resource_dict, property_name, pare If path is already a path to S3 object, this method does nothing. :param resource_id: Id of the CloudFormation resource - :param resource_dict: Dictionary containing resource definition + :param asset: Iac ImageAsset binded to the resource :param property_name: Property name of CloudFormation resource where this local path is present :param parent_dir: Resolve all relative paths with respect to this @@ -72,7 +73,7 @@ def upload_local_image_artifacts(resource_id, resource_dict, property_name, pare :return: ECR URL of the uploaded object """ - image_path = jmespath.search(property_name, resource_dict) + image_path = asset.source_local_image if not image_path: raise ImageNotFoundError(property_name=property_name, resource_id=resource_id) @@ -86,7 +87,7 @@ def upload_local_image_artifacts(resource_id, resource_dict, property_name, pare def upload_local_artifacts( resource_id: str, - resource_dict: Dict, + asset: IacS3Asset, property_name: str, parent_dir: str, uploader: S3Uploader, @@ -105,7 +106,7 @@ def upload_local_artifacts( If path is already a path to S3 object, this method does nothing. :param resource_id: Id of the CloudFormation resource - :param resource_dict: Dictionary containing resource definition + :param asset: Iac S3Asset binded to the resource :param property_name: Property name of CloudFormation resource where this local path is present :param parent_dir: Resolve all relative paths with respect to this @@ -116,9 +117,9 @@ def upload_local_artifacts( :raise: ValueError if path is not a S3 URL or a local path """ - local_path = jmespath.search(property_name, resource_dict) + local_path = asset.source_path or "" - if local_path is None: + if not local_path: # Build the root directory and upload to S3 local_path = parent_dir @@ -128,7 +129,7 @@ def upload_local_artifacts( # refer to local artifacts # Nothing to do if property value is an S3 URL LOG.debug("Property %s of %s is already a S3 URL", property_name, resource_id) - return cast(str, local_path) + return local_path local_path = make_abs_path(parent_dir, local_path) diff --git a/samcli/lib/providers/sam_base_provider.py b/samcli/lib/providers/sam_base_provider.py index 9af3794801..29623adb18 100644 --- a/samcli/lib/providers/sam_base_provider.py +++ b/samcli/lib/providers/sam_base_provider.py @@ -116,7 +116,10 @@ def _extract_lambda_function_imageuri(resource_properties: Dict, code_property_k str Representing the local imageuri """ - return cast(Optional[str], resource_properties.get(code_property_key, dict()).get("ImageUri", None)) + code = resource_properties.get(code_property_key, dict()) + if isinstance(code, str): + return cast(Optional[str], code) + return cast(Optional[str], code.get("ImageUri", None)) @staticmethod def _extract_sam_function_imageuri(resource_properties: Dict, code_property_key: str) -> Optional[str]: @@ -138,7 +141,11 @@ def _extract_sam_function_imageuri(resource_properties: Dict, code_property_key: return resource_properties.get(code_property_key, None) @staticmethod - def get_template(template_dict: IacStack, parameter_overrides: Optional[Dict[str, str]] = None) -> IacStack: + def get_template( + template_dict: IacStack, + parameter_overrides: Optional[Dict[str, str]] = None, + normalize_resource_metadata: bool = True, + ) -> IacStack: """ Given a SAM template dictionary, return a cleaned copy of the template where SAM plugins have been run and parameter values have been substituted. @@ -151,6 +158,10 @@ def get_template(template_dict: IacStack, parameter_overrides: Optional[Dict[str parameter_overrides: dict Optional dictionary of values for template parameters + normalize_resource_metadata: bool + flag to normalize resource metadata or not; For package and deploy, we don't need to normalize resource + metadata, which usually exists in a CDK-synthed template and is used for build and local testing + Returns ------- dict @@ -160,7 +171,8 @@ def get_template(template_dict: IacStack, parameter_overrides: Optional[Dict[str parameters_values = SamBaseProvider._get_parameter_values(template_dict, parameter_overrides) if template_dict: template_dict = SamTranslatorWrapper(template_dict, parameter_values=parameters_values).run_plugins() - ResourceMetadataNormalizer.normalize(template_dict) + if normalize_resource_metadata: + ResourceMetadataNormalizer.normalize(template_dict) resolver = IntrinsicResolver( template=template_dict, diff --git a/samcli/lib/providers/sam_stack_provider.py b/samcli/lib/providers/sam_stack_provider.py index 27b21a0011..7ddcea27b0 100644 --- a/samcli/lib/providers/sam_stack_provider.py +++ b/samcli/lib/providers/sam_stack_provider.py @@ -7,7 +7,7 @@ from typing import Optional, Dict, cast, List, Iterator, Tuple from urllib.parse import unquote, urlparse -from samcli.lib.iac.interface import Stack as IacStack, Resource, S3Asset +from samcli.lib.iac.interface import Stack as IacStack, Resource as IacResource, S3Asset from samcli.lib.providers.exceptions import RemoteStackLocationNotSupported from samcli.lib.providers.provider import Stack, get_full_path from samcli.lib.providers.sam_base_provider import SamBaseProvider @@ -29,6 +29,7 @@ def __init__( template_dict: IacStack, parameter_overrides: Optional[Dict] = None, global_parameter_overrides: Optional[Dict] = None, + normalize_resource_metadata: bool = True, ): """ Initialize the class with SAM template data. The SAM template passed to this provider is assumed @@ -45,6 +46,8 @@ def __init__( to get substituted within the template :param dict global_parameter_overrides: Optional dictionary of values for SAM template global parameters that might want to get substituted within the template and all its child templates + :param bool normalize_resource_metadata: should normalize resource metadata or not. Resource metadata usually + exists in CDK-synted templates and is used for build and local testing """ self._stack_origin_dir = stack_origin_dir @@ -52,6 +55,7 @@ def __init__( self._template_dict: IacStack = self.get_template( template_dict, SamLocalStackProvider.merge_parameter_overrides(parameter_overrides, global_parameter_overrides), + normalize_resource_metadata=normalize_resource_metadata, ) self._resources = self._template_dict.get("Resources", {}) self._global_parameter_overrides = global_parameter_overrides @@ -132,7 +136,7 @@ def _convert_sam_application_resource( stack_origin_dir: str, stack_path: str, name: str, - resource: Resource, + resource: IacResource, resource_properties: Dict, global_parameter_overrides: Optional[Dict] = None, ) -> Optional[Stack]: @@ -170,7 +174,7 @@ def _convert_cfn_stack_resource( stack_origin_dir: str, stack_path: str, name: str, - resource: Resource, + resource: IacResource, resource_properties: Dict, global_parameter_overrides: Optional[Dict] = None, ) -> Optional[Stack]: @@ -206,6 +210,7 @@ def get_stacks( name: str = "", parameter_overrides: Optional[Dict] = None, global_parameter_overrides: Optional[Dict] = None, + normalize_resource_metadata: bool = True, ) -> Tuple[List[Stack], List[str]]: """ Recursively extract stacks from a template file. @@ -224,6 +229,9 @@ def get_stacks( global_parameter_overrides: Optional[Dict] Optional dictionary of values for SAM template global parameters that might want to get substituted within the template and its child templates + normalize_resource_metadata: bool + boolean flag to normalize resource metadata or not. Resource metadata usually + exists in CDK-synted templates and is used for build and local testing Returns ------- @@ -247,7 +255,12 @@ def get_stacks( ) current = SamLocalStackProvider( - project_stack.origin_dir, stack_path, project_stack, parameter_overrides, global_parameter_overrides + project_stack.origin_dir, + stack_path, + project_stack, + parameter_overrides, + global_parameter_overrides, + normalize_resource_metadata, ) remote_stack_full_paths.extend(current.remote_stack_full_paths) @@ -258,6 +271,7 @@ def get_stacks( child_stack.logical_id, child_stack.parameters, global_parameter_overrides, + normalize_resource_metadata, ) stacks.extend(stacks_in_child) remote_stack_full_paths.extend(remote_stack_full_paths_in_child) diff --git a/samcli/lib/samlib/resource_metadata_normalizer.py b/samcli/lib/samlib/resource_metadata_normalizer.py index a200e9f479..d2a34807cc 100644 --- a/samcli/lib/samlib/resource_metadata_normalizer.py +++ b/samcli/lib/samlib/resource_metadata_normalizer.py @@ -62,6 +62,7 @@ def replace_property(property_key, property_value, resource, logical_id): while len(nested_keys) > 1: key = nested_keys.pop(0) target_dict[key] = {} + target_dict = target_dict[key] target_dict[nested_keys[0]] = property_value elif property_key or property_value: LOG.info( diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index ae5fb65544..654441c71e 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -87,22 +87,25 @@ class TestBuildCommand_PythonFunctions(BuildIntegBase): @parameterized.expand( [ - ("python2.7", False), - ("python3.6", False), - ("python3.7", False), - ("python3.8", False), - ("python2.7", "use_container"), - ("python3.6", "use_container"), - ("python3.7", "use_container"), - ("python3.8", "use_container"), + ("python2.7", "Python", False), + ("python3.6", "Python", False), + ("python3.7", "Python", False), + ("python3.8", "Python", False), + # numpy 1.20.3 (in PythonPEP600/requirements.txt) only support python 3.7+ + ("python3.7", "PythonPEP600", False), + ("python3.8", "PythonPEP600", False), + ("python2.7", "Python", "use_container"), + ("python3.6", "Python", "use_container"), + ("python3.7", "Python", "use_container"), + ("python3.8", "Python", "use_container"), ] ) @pytest.mark.flaky(reruns=3) - def test_with_default_requirements(self, runtime, use_container): + def test_with_default_requirements(self, runtime, codeuri, use_container): if use_container and SKIP_DOCKER_TESTS: self.skipTest(SKIP_DOCKER_MESSAGE) - overrides = {"Runtime": runtime, "CodeUri": "Python", "Handler": "main.handler"} + overrides = {"Runtime": runtime, "CodeUri": codeuri, "Handler": "main.handler"} cmdlist = self.get_command_list(use_container=use_container, parameter_overrides=overrides) LOG.info("Running Command: {}".format(cmdlist)) @@ -332,7 +335,11 @@ def _prepare_application_environment(self): class TestBuildCommand_Java(BuildIntegBase): EXPECTED_FILES_PROJECT_MANIFEST_GRADLE = {"aws", "lib", "META-INF"} EXPECTED_FILES_PROJECT_MANIFEST_MAVEN = {"aws", "lib"} - EXPECTED_DEPENDENCIES = {"annotations-2.1.0.jar", "aws-lambda-java-core-1.1.0.jar"} + EXPECTED_GRADLE_DEPENDENCIES = {"annotations-2.1.0.jar", "aws-lambda-java-core-1.1.0.jar"} + EXPECTED_MAVEN_DEPENDENCIES = { + "software.amazon.awssdk.annotations-2.1.0.jar", + "com.amazonaws.aws-lambda-java-core-1.1.0.jar", + } FUNCTION_LOGICAL_ID = "Function" USING_GRADLE_PATH = os.path.join("Java", "gradle") @@ -342,60 +349,70 @@ class TestBuildCommand_Java(BuildIntegBase): @parameterized.expand( [ - ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), - ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), - ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), - ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, EXPECTED_MAVEN_DEPENDENCIES), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8.al2", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ( + "java8.al2", + USING_GRADLE_KOTLIN_PATH, + EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, + EXPECTED_GRADLE_DEPENDENCIES, + ), + ("java8.al2", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, EXPECTED_MAVEN_DEPENDENCIES), + ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, EXPECTED_MAVEN_DEPENDENCIES), + ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), ] ) @skipIf(SKIP_DOCKER_TESTS, SKIP_DOCKER_MESSAGE) @pytest.mark.flaky(reruns=3) - def test_building_java_in_container(self, runtime, code_path, expected_files): - self._test_with_building_java(runtime, code_path, expected_files, "use_container") + def test_building_java_in_container(self, runtime, code_path, expected_files, expected_dependencies): + self._test_with_building_java(runtime, code_path, expected_files, expected_dependencies, "use_container") @parameterized.expand( [ - ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), - ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java8.al2", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), - ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, EXPECTED_MAVEN_DEPENDENCIES), + ("java8", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java8.al2", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ( + "java8.al2", + USING_GRADLE_KOTLIN_PATH, + EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, + EXPECTED_GRADLE_DEPENDENCIES, + ), + ("java8.al2", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, EXPECTED_MAVEN_DEPENDENCIES), + ("java8.al2", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), ] ) @pytest.mark.flaky(reruns=3) - def test_building_java8_in_process(self, runtime, code_path, expected_files): - self._test_with_building_java(runtime, code_path, expected_files, False) + def test_building_java8_in_process(self, runtime, code_path, expected_files, expected_dependencies): + self._test_with_building_java(runtime, code_path, expected_files, expected_dependencies, False) @parameterized.expand( [ - ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), - ("java11", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN), - ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE), + ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_GRADLEW_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_GRADLE_KOTLIN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), + ("java11", USING_MAVEN_PATH, EXPECTED_FILES_PROJECT_MANIFEST_MAVEN, EXPECTED_MAVEN_DEPENDENCIES), + ("java11", USING_GRADLE_PATH, EXPECTED_FILES_PROJECT_MANIFEST_GRADLE, EXPECTED_GRADLE_DEPENDENCIES), ] ) @pytest.mark.flaky(reruns=3) - def test_building_java11_in_process(self, runtime, code_path, expected_files): - self._test_with_building_java(runtime, code_path, expected_files, False) + def test_building_java11_in_process(self, runtime, code_path, expected_files, expected_dependencies): + self._test_with_building_java(runtime, code_path, expected_files, expected_dependencies, False) - def _test_with_building_java(self, runtime, code_path, expected_files, use_container): + def _test_with_building_java(self, runtime, code_path, expected_files, expected_dependencies, use_container): if use_container and SKIP_DOCKER_TESTS: self.skipTest(SKIP_DOCKER_MESSAGE) @@ -409,7 +426,7 @@ def _test_with_building_java(self, runtime, code_path, expected_files, use_conta run_command(cmdlist, cwd=self.working_dir) self._verify_built_artifact( - self.default_build_dir, self.FUNCTION_LOGICAL_ID, expected_files, self.EXPECTED_DEPENDENCIES + self.default_build_dir, self.FUNCTION_LOGICAL_ID, expected_files, expected_dependencies ) self._verify_resource_property( diff --git a/tests/integration/testdata/buildcmd/PythonPEP600/__init__.py b/tests/integration/testdata/buildcmd/PythonPEP600/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/testdata/buildcmd/PythonPEP600/main.py b/tests/integration/testdata/buildcmd/PythonPEP600/main.py new file mode 100644 index 0000000000..b636d9d592 --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonPEP600/main.py @@ -0,0 +1,19 @@ +import numpy + + +# from cryptography.fernet import Fernet + + +def handler(event, context): + # Try using some of the modules to make sure they work & don't crash the process + # print(Fernet.generate_key()) + + return {"pi": "{0:.2f}".format(numpy.pi)} + + +def first_function_handler(event, context): + return "Hello World" + + +def second_function_handler(event, context): + return "Hello Mars" diff --git a/tests/integration/testdata/buildcmd/PythonPEP600/requirements.txt b/tests/integration/testdata/buildcmd/PythonPEP600/requirements.txt new file mode 100644 index 0000000000..a58b87977c --- /dev/null +++ b/tests/integration/testdata/buildcmd/PythonPEP600/requirements.txt @@ -0,0 +1,6 @@ +# These are some hard packages to build. Using them here helps us verify that building works on various platforms + +# these dependency versions use PEP600 +numpy==1.20.3 +greenlet==1.1.0 +sqlalchemy==1.4.15 diff --git a/tests/unit/commands/_utils/test_iac_validations.py b/tests/unit/commands/_utils/test_iac_validations.py new file mode 100644 index 0000000000..39f09e1102 --- /dev/null +++ b/tests/unit/commands/_utils/test_iac_validations.py @@ -0,0 +1,153 @@ +from unittest import TestCase +from unittest.mock import patch, MagicMock, Mock + +import click + +from samcli.commands._utils.iac_validations import iac_options_validation + + +def _make_ctx_params_side_effect_func(params): + def side_effect(key, default=None): + return params.get(key, default) + + return side_effect + + +class TestIacValidations(TestCase): + def setUp(self): + @iac_options_validation(require_stack=False) + def func_not_require_stack(*args, **kwargs): + pass + + @iac_options_validation(require_stack=True) + def func_require_stack(*args, **kwargs): + pass + + self.func_require_stack = func_require_stack + self.func_not_require_stack = func_not_require_stack + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_success_cfn_not_require_stack(self, click_mock): + params = {"project_type": "CFN"} + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + + project_mock = Mock() + self.func_not_require_stack(project=project_mock) + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_success_cfn_require_stack(self, click_mock): + params = {"project_type": "CFN"} + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + + project_mock = Mock() + self.func_require_stack(project=project_mock) + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_fail_cfn_invalid_options(self, click_mock): + params = { + "project_type": "CFN", + "cdk_app": "foo", + } + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + with self.assertRaises(click_mock.BadOptionUsage) as ex: + self.func_require_stack(project=project_mock) + self.assertEqual(ex.exception.option_name, "--cdk-app") + self.assertEqual(ex.exception.message, "Option '--cdk-app' cannot be used for Project Type 'CFN'") + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_success_cdk_not_require_stack(self, click_mock): + params = { + "project_type": "CDK", + "cdk_app": "foo", + } + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + stack_mock = Mock() + stack_mock.name = "stack" + project_mock.stacks = [stack_mock] + + self.func_not_require_stack(project=project_mock) + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_success_cdk_require_stack(self, click_mock): + params = {"project_type": "CDK", "cdk_app": "foo", "stack_name": "stack"} + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + stack_mock = Mock() + stack_mock.name = "stack" + project_mock.stacks = [stack_mock] + + self.func_require_stack(project=project_mock) + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_success_cdk_no_need_to_specify_stack_name(self, click_mock): + params = { + "project_type": "CDK", + "cdk_app": "foo", + } + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + stack_mock = Mock() + stack_mock.name = "stack" + project_mock.stacks = [stack_mock] + + self.func_require_stack(project=project_mock) + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_fail_cdk_missing_stack_name(self, click_mock): + params = { + "project_type": "CDK", + "cdk_app": "foo", + } + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + project_mock.stacks = [Mock(), Mock()] + + with self.assertRaises(click_mock.BadOptionUsage) as ex: + self.func_require_stack(project=project_mock) + self.assertEqual(ex.exception.option_name, "--stack-name") + self.assertEqual(ex.exception.message, "More than one stack found. Use '--stack-name' to specify the stack.") + + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_fail_cdk_not_found_stack_name(self, click_mock): + params = {"project_type": "CDK", "cdk_app": "foo", "stack_name": "non_existent_stack"} + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + stack_mock = Mock() + stack_mock.name = "stack" + project_mock.stacks = [stack_mock] + project_mock.find_stack_by_name.return_value = None + + with self.assertRaises(click_mock.BadOptionUsage) as ex: + self.func_require_stack(project=project_mock) + self.assertEqual(ex.exception.option_name, "--stack-name") + self.assertEqual(ex.exception.message, "Stack with stack name 'non_existent_stack' not found.") diff --git a/tests/unit/commands/_utils/test_options.py b/tests/unit/commands/_utils/test_options.py index bbc6701049..592972efb6 100644 --- a/tests/unit/commands/_utils/test_options.py +++ b/tests/unit/commands/_utils/test_options.py @@ -13,10 +13,12 @@ from tomlkit import parse from samcli.commands._utils.options import ( + determine_project_type, get_or_default_template_file_name, _TEMPLATE_OPTION_DEFAULT_VALUE, guided_deploy_stack_name, artifact_callback, + project_type_callback, resolve_s3_callback, image_repositories_callback, _space_separated_list_func_type, @@ -452,3 +454,65 @@ class TestSpaceSeparatedListInvalidDataTypes: def test_raise_value_error(self, test_input): with pytest.raises(ValueError): _space_separated_list_func_type(test_input) + + +class TestProjectTypeCallback(TestCase): + @patch("samcli.commands._utils.options.determine_project_type") + def test_raise_error_if_detected_not_match_inputted(self, determine_project_type_mock): + context_mock = MockContext(info_name="test", parent=None, params=Mock()) + param_mock = Mock() + param_mock.name = "--project-type" + provided_value = "CDK" + detected_value = "CFN" + include_build = True + determine_project_type_mock.return_value = detected_value + with self.assertRaises(click.BadOptionUsage) as ex: + project_type_callback(context_mock, param_mock, provided_value, include_build) + self.assertEqual(ex.exception.option_name, param_mock.name) + self.assertEqual( + ex.exception.message, "It seems your project type is CFN. However, you specified CDK in --project-type" + ) + + @patch("samcli.commands._utils.options.determine_project_type") + def test_return_detected_project_type(self, determine_project_type_mock): + context_mock = MockContext(info_name="test", parent=None, params=Mock()) + param_mock = Mock() + param_mock.name = "--project-type" + detected_value = "CFN" + include_build = True + determine_project_type_mock.return_value = detected_value + self.assertEqual(project_type_callback(context_mock, param_mock, None, include_build), detected_value) + + @patch("samcli.commands._utils.options.determine_project_type") + def test_return_provided_project_type(self, determine_project_type_mock): + context_mock = MockContext(info_name="test", parent=None, params=Mock()) + param_mock = Mock() + param_mock.name = "--project-type" + detected_value = "CFN" + provided_value = "CFN" + include_build = True + determine_project_type_mock.return_value = detected_value + self.assertEqual(project_type_callback(context_mock, param_mock, provided_value, include_build), provided_value) + + +class TestDetermineProjectType(TestCase): + @patch("samcli.commands._utils.options.find_cfn_template") + @patch("samcli.commands._utils.options.find_cdk_file") + def test_return_cfn_type(self, find_cdk_file_mock, find_cfn_template_mock): + find_cfn_template_mock.return_value = True + find_cdk_file_mock.return_value = False + self.assertEqual(determine_project_type(True), "CFN") + + @patch("samcli.commands._utils.options.find_cfn_template") + @patch("samcli.commands._utils.options.find_cdk_file") + def test_return_cdk_type(self, find_cdk_file_mock, find_cfn_template_mock): + find_cfn_template_mock.return_value = False + find_cdk_file_mock.return_value = True + self.assertEqual(determine_project_type(True), "CDK") + + @patch("samcli.commands._utils.options.find_cfn_template") + @patch("samcli.commands._utils.options.find_cdk_file") + def test_return_default(self, find_cdk_file_mock, find_cfn_template_mock): + find_cfn_template_mock.return_value = False + find_cdk_file_mock.return_value = False + self.assertEqual(determine_project_type(True), "CFN") diff --git a/tests/unit/commands/deploy/test_command.py b/tests/unit/commands/deploy/test_command.py index 73d2973725..ab124491ca 100644 --- a/tests/unit/commands/deploy/test_command.py +++ b/tests/unit/commands/deploy/test_command.py @@ -47,6 +47,13 @@ def setUp(self): self.config_env = "mock-default-env" self.config_file = "mock-default-filename" self.signing_profiles = None + self.project_type = "CFN" + self.project = MagicMock() + self.iac = Mock() + self.stack_name = "" + self.iac_stack_mock = MagicMock() + self.project.find_stack_by_name.return_value = self.iac_stack_mock + self.project.stacks.__getitem__.return_value = self.iac_stack_mock MOCK_SAM_CONFIG.reset_mock() @patch("samcli.commands.package.command.click") @@ -85,6 +92,9 @@ def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_con resolve_s3=self.resolve_s3, config_env=self.config_env, config_file=self.config_file, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) mock_deploy_context.assert_called_with( @@ -119,25 +129,21 @@ def test_all_args(self, mock_deploy_context, mock_deploy_click, mock_package_con @patch("samcli.commands.deploy.deploy_context.DeployContext") @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") - @patch("samcli.commands.deploy.guided_context.get_template_parameters") + # @patch("samcli.commands.deploy.guided_context.get_template_parameters") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") @patch.object(GuidedConfig, "get_config_ctx", MagicMock(return_value=(None, get_mock_sam_config()))) @patch("samcli.commands.deploy.guided_context.prompt") @patch("samcli.commands.deploy.guided_context.confirm") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_all_args_guided_no_to_authorization_confirmation_prompt( self, - mock_get_template_data, mock_confirm, mock_prompt, mock_signer_config_per_function, mock_sam_function_provider, - mock_get_template_artifacts_format, mock_get_buildable_stacks, - mock_get_template_parameters, + # mock_get_template_parameters, mockauth_per_resource, mock_managed_stack, mock_deploy_context, @@ -145,10 +151,13 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( mock_package_context, mock_package_click, ): - mock_get_template_data.return_value = {} + self.iac_stack_mock.has_assets_of_package_type.return_value = False + self.iac_stack_mock.get_overrideable_parameters.return_value = { + "Myparameter": {"Type": "String"}, + "MyNoEchoParameter": {"Type": "String", "NoEcho": True}, + } mock_get_buildable_stacks.return_value = (Mock(), []) mock_sam_function_provider.return_value = {} - mock_get_template_artifacts_format.return_value = [ZIP] context_mock = Mock() mockauth_per_resource.return_value = [("HelloWorldResource1", False), ("HelloWorldResource2", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock @@ -163,11 +172,6 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( "test-env", ] - mock_get_template_parameters.return_value = { - "Myparameter": {"Type": "String"}, - "MyNoEchoParameter": {"Type": "String", "NoEcho": True}, - } - mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -200,6 +204,9 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( resolve_s3=self.resolve_s3, config_env=self.config_env, config_file=self.config_file, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) @patch("samcli.commands.package.command.click") @@ -208,29 +215,21 @@ def test_all_args_guided_no_to_authorization_confirmation_prompt( @patch("samcli.commands.deploy.deploy_context.DeployContext") @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") - @patch("samcli.commands.deploy.guided_context.get_template_parameters") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") @patch.object(GuidedConfig, "get_config_ctx", MagicMock(return_value=(None, get_mock_sam_config()))) @patch("samcli.commands.deploy.guided_context.prompt") @patch("samcli.commands.deploy.guided_context.confirm") - @patch("samcli.commands.deploy.guided_context.get_template_data") @patch("samcli.commands.deploy.guided_context.tag_translation") def test_all_args_guided( self, mock_tag_translation, - mock_get_template_data, mock_confirm, mock_prompt, mock_signer_config_per_function, mock_sam_function_provider, - mock_get_template_function_resource_ids, - mock_get_template_artifacts_format, mock_get_buildable_stacks, - mock_get_template_parameters, mockauth_per_resource, mock_managed_stack, mock_deploy_context, @@ -238,16 +237,21 @@ def test_all_args_guided( mock_package_context, mock_package_click, ): - mock_get_template_data.return_value = {} + self.iac_stack_mock.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.iac_stack_mock.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.iac_stack_mock.get_overrideable_parameters.return_value = { + "Myparameter": {"Type": "String"}, + "MyNoEchoParameter": {"Type": "String", "NoEcho": True}, + } mock_get_buildable_stacks.return_value = (Mock(), []) mock_tag_translation.return_value = "helloworld-123456-v1" - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] context_mock = Mock() mock_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri="helloworld:v1")} ) - mock_get_template_artifacts_format.return_value = [IMAGE] mockauth_per_resource.return_value = [("HelloWorldResource", False)] mock_deploy_context.return_value.__enter__.return_value = context_mock mock_confirm.side_effect = [True, False, True, True] @@ -262,11 +266,6 @@ def test_all_args_guided( "test-env", ] - mock_get_template_parameters.return_value = { - "Myparameter": {"Type": "String"}, - "MyNoEchoParameter": {"Type": "String", "NoEcho": True}, - } - mock_managed_stack.return_value = "managed-s3-bucket" mock_signer_config_per_function.return_value = ({}, {}) @@ -299,6 +298,9 @@ def test_all_args_guided( resolve_s3=self.resolve_s3, config_env=self.config_env, config_file=self.config_file, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) mock_deploy_context.assert_called_with( @@ -352,9 +354,6 @@ def test_all_args_guided( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_parameters") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") @patch.object( @@ -364,19 +363,14 @@ def test_all_args_guided( ) @patch("samcli.commands.deploy.guided_context.prompt") @patch("samcli.commands.deploy.guided_context.confirm") - @patch("samcli.commands.deploy.guided_context.get_template_data") @patch("samcli.commands.deploy.guided_context.tag_translation") def test_all_args_guided_no_save_echo_param_to_config( self, mock_tag_translation, - mock_get_template_data, mock_confirm, mock_prompt, mock_signer_config_per_function, mock_sam_function_provider, - mock_get_template_artifacts_format, - mock_get_template_function_resource_ids, - mock_get_template_parameters, mock_get_buildable_stacks, mockauth_per_resource, mock_managed_stack, @@ -385,22 +379,23 @@ def test_all_args_guided_no_save_echo_param_to_config( mock_package_context, mock_package_click, ): - mock_get_template_data.return_value = {} + self.iac_stack_mock.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.iac_stack_mock.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.iac_stack_mock.get_overrideable_parameters.return_value = { + "Myparameter": {"Type": "String"}, + "MyParameterSpaces": {"Type": "String"}, + "MyNoEchoParameter": {"Type": "String", "NoEcho": True}, + } mock_get_buildable_stacks.return_value = (Mock(), []) mock_tag_translation.return_value = "helloworld-123456-v1" - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] context_mock = Mock() mock_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri="helloworld:v1")} ) - mock_get_template_artifacts_format.return_value = [IMAGE] mockauth_per_resource.return_value = [("HelloWorldResource", False)] - mock_get_template_parameters.return_value = { - "Myparameter": {"Type": "String"}, - "MyParameterSpaces": {"Type": "String"}, - "MyNoEchoParameter": {"Type": "String", "NoEcho": True}, - } mock_deploy_context.return_value.__enter__.return_value = context_mock mock_prompt.side_effect = [ "sam-app", @@ -445,6 +440,9 @@ def test_all_args_guided_no_save_echo_param_to_config( resolve_s3=self.resolve_s3, config_env=self.config_env, config_file=self.config_file, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) mock_deploy_context.assert_called_with( @@ -512,10 +510,7 @@ def test_all_args_guided_no_save_echo_param_to_config( @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") @patch("samcli.commands.deploy.guided_context.manage_stack") - @patch("samcli.commands.deploy.guided_context.get_template_parameters") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch.object( GuidedConfig, @@ -526,21 +521,16 @@ def test_all_args_guided_no_save_echo_param_to_config( @patch("samcli.commands.deploy.guided_context.confirm") @patch("samcli.commands.deploy.guided_config.SamConfig") @patch("samcli.commands.deploy.guided_config.get_cmd_names") - @patch("samcli.commands.deploy.guided_context.get_template_data") @patch("samcli.commands.deploy.guided_context.tag_translation") def test_all_args_guided_no_params_save_config( self, mock_tag_translation, - mock_get_template_data, mock_get_cmd_names, mock_sam_config, mock_confirm, mock_prompt, mock_sam_function_provider, - mock_get_template_function_resource_ids, - mock_get_template_artifacts_format, mock_signer_config_per_function, - mock_get_template_parameters, mock_managed_stack, mock_get_buildable_stacks, mockauth_per_resource, @@ -549,19 +539,20 @@ def test_all_args_guided_no_params_save_config( mock_package_context, mock_package_click, ): - mock_get_template_data.return_value = {} + self.iac_stack_mock.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.iac_stack_mock.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.iac_stack_mock.get_overrideable_parameters.return_value = {} mock_get_buildable_stacks.return_value = (Mock(), []) mock_tag_translation.return_value = "helloworld-123456-v1" - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] context_mock = Mock() mock_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri="helloworld:v1")} ) - mock_get_template_artifacts_format.return_value = [IMAGE] mockauth_per_resource.return_value = [("HelloWorldResource", False)] - mock_get_template_parameters.return_value = {} mock_deploy_context.return_value.__enter__.return_value = context_mock mock_prompt.side_effect = [ "sam-app", @@ -603,6 +594,9 @@ def test_all_args_guided_no_params_save_config( config_env=self.config_env, config_file=self.config_file, signing_profiles=self.signing_profiles, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) mock_deploy_context.assert_called_with( @@ -660,27 +654,19 @@ def test_all_args_guided_no_params_save_config( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_parameters") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") @patch.object(GuidedConfig, "get_config_ctx", MagicMock(return_value=(None, get_mock_sam_config()))) @patch("samcli.commands.deploy.guided_context.prompt") @patch("samcli.commands.deploy.guided_context.confirm") - @patch("samcli.commands.deploy.guided_context.get_template_data") @patch("samcli.commands.deploy.guided_context.tag_translation") def test_all_args_guided_no_params_no_save_config( self, mock_tag_translation, - mock_get_template_data, mock_confirm, mock_prompt, mock_signer_config_per_function, mock_sam_function_provider, - mock_get_template_artifacts_format, - mock_get_template_function_resource_ids, - mock_get_template_parameters, mock_get_buildable_stacks, mockauth_per_resource, mock_managed_stack, @@ -689,18 +675,19 @@ def test_all_args_guided_no_params_no_save_config( mock_package_context, mock_package_click, ): - mock_get_template_data.return_value = {} + self.iac_stack_mock.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.iac_stack_mock.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.iac_stack_mock.get_overrideable_parameters.return_value = {} mock_get_buildable_stacks.return_value = (Mock(), []) mock_tag_translation.return_value = "helloworld-123456-v1" - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] context_mock = Mock() mock_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri="helloworld:v1")} ) - mock_get_template_artifacts_format.return_value = [IMAGE] mockauth_per_resource.return_value = [("HelloWorldResource", False)] - mock_get_template_parameters.return_value = {} mock_deploy_context.return_value.__enter__.return_value = context_mock mock_prompt.side_effect = [ "sam-app", @@ -742,6 +729,9 @@ def test_all_args_guided_no_params_no_save_config( config_file=self.config_file, config_env=self.config_env, signing_profiles=self.signing_profiles, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) mock_deploy_context.assert_called_with( @@ -811,6 +801,9 @@ def test_all_args_resolve_s3( config_file=self.config_file, config_env=self.config_env, signing_profiles=self.signing_profiles, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) mock_deploy_context.assert_called_with( @@ -868,4 +861,7 @@ def test_resolve_s3_and_s3_bucket_both_set(self): config_file=self.config_file, config_env=self.config_env, signing_profiles=self.signing_profiles, + project_type=self.project_type, + project=self.project, + iac=self.iac, ) diff --git a/tests/unit/commands/deploy/test_guided_context.py b/tests/unit/commands/deploy/test_guided_context.py index 7025f2f0c4..e6932bb2d5 100644 --- a/tests/unit/commands/deploy/test_guided_context.py +++ b/tests/unit/commands/deploy/test_guided_context.py @@ -11,8 +11,10 @@ class TestGuidedContext(TestCase): def setUp(self): + iac_mock = Mock() + self.project_mock = project_mock = Mock() + project_mock.stacks = MagicMock() self.gc = GuidedContext( - template_file="template", stack_name="test", s3_bucket="s3_b", s3_prefix="s3_p", @@ -20,32 +22,92 @@ def setUp(self): region="region", image_repository=None, image_repositories={"HelloWorldFunction": "image-repo"}, + iac=iac_mock, + project=self.project_mock, ) + def test_get_iac_stack_stack_name_found(self): + project_mock = self.gc._project + project_mock.reset_mock() + stack_mock = Mock() + stack_mock.origin_dir = "dir" + stack_mock.name = "stack_name" + project_mock.find_stack_by_name.return_value = stack_mock + + self.gc.stack_name = "test" + self.gc._get_iac_stack() + self.assertEqual(self.gc._iac_stack, stack_mock) + self.assertEqual(self.gc.template_file, "dir") + self.assertEqual(self.gc.stack_name, "test") + + def test_get_iac_stack_stack_name_not_found(self): + project_mock = self.gc._project + project_mock.reset_mock() + stack_mock = Mock() + stack_mock.origin_dir = "dir" + stack_mock.name = "stack_name" + project_mock.find_stack_by_name.return_value = None + + self.gc.stack_name = "test" + with self.assertRaises(GuidedDeployFailedError) as ex: + self.gc._get_iac_stack() + self.assertEqual( + ex.exception.msg, + "There is no stack with name 'test'. " + "If you have specified --stack-name, specify the correct stack name or remove --stack-name to use default.", + ) + + def test_get_iac_stack_stack_name_none(self): + project_mock = self.gc._project + project_mock.reset_mock() + stack_mock = Mock() + stack_mock.origin_dir = "dir" + stack_mock.name = "stack_name" + project_mock.stacks.__getitem__.return_value = stack_mock + + self.gc.stack_name = None + self.gc._get_iac_stack() + self.assertEqual(self.gc._iac_stack, stack_mock) + self.assertEqual(self.gc.template_file, "dir") + self.assertEqual(self.gc.stack_name, "stack_name") + + def test_get_iac_stack_stack_name_empty_sting(self): + project_mock = self.gc._project + project_mock.reset_mock() + stack_mock = Mock() + stack_mock.origin_dir = "dir" + stack_mock.name = "" + project_mock.stacks.__getitem__.return_value = stack_mock + + self.gc.stack_name = None + self.gc._get_iac_stack() + self.assertEqual(self.gc._iac_stack, stack_mock) + self.assertEqual(self.gc.template_file, "dir") + self.assertEqual(self.gc.stack_name, "sam-app") + self.gc.stack_name = "test" + @patch("samcli.commands.deploy.guided_context.prompt") @patch("samcli.commands.deploy.guided_context.confirm") @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_defaults_non_public_resources_zips( self, - patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [ @@ -54,7 +116,7 @@ def test_guided_prompts_check_defaults_non_public_resources_zips( patched_confirm.side_effect = [True, False, "", True] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -76,32 +138,34 @@ def test_guided_prompts_check_defaults_non_public_resources_zips( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") + # @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") + # @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_defaults_public_resources_zips( self, - patched_get_template_data, + # patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, + # patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_signer_config_per_function.return_value = (None, None) patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_confirm.side_effect = [True, False, True, False, ""] patched_manage_stack.return_value = "managed_s3_stack" - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -127,36 +191,34 @@ def test_guided_prompts_check_defaults_public_resources_zips( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.click.secho") @patch("samcli.commands.deploy.guided_context.tag_translation") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_defaults_public_resources_images( self, - patched_get_template_data, patched_signer_config_per_function, patched_tag_translation, patched_click_secho, patched_sam_function_provider, - patched_get_template_artifacts_format, - mock_get_template_function_resource_ids, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.gc._iac_stack.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_signer_config_per_function.return_value = (None, None) patched_tag_translation.return_value = "helloworld-123456-v1" patched_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri="helloworld:v1")} ) - patched_get_template_artifacts_format.return_value = [IMAGE] patched_get_buildable_stacks.return_value = (Mock(), []) patched_prompt.side_effect = [ "sam-app", @@ -168,7 +230,7 @@ def test_guided_prompts_check_defaults_public_resources_images( patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_confirm.side_effect = [True, False, True, False, ""] patched_manage_stack.return_value = "managed_s3_stack" - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -210,28 +272,29 @@ def test_guided_prompts_check_defaults_public_resources_images( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") + # @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") + # @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.click.secho") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") + # @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_defaults_public_resources_images_ecr_url( self, - patched_get_template_data, patched_signer_config_per_function, patched_click_secho, patched_sam_function_provider, - mock_get_template_function_resource_ids, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.gc._iac_stack.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.gc._iac_stack.get_overrideable_parameters.return_value = {} patched_sam_function_provider.return_value = MagicMock( functions={ @@ -240,7 +303,6 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( ) } ) - patched_get_template_artifacts_format.return_value = [IMAGE] patched_get_buildable_stacks.return_value = (Mock(), []) patched_prompt.side_effect = [ "sam-app", @@ -253,7 +315,7 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( patched_confirm.side_effect = [True, False, True, False, ""] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -290,34 +352,31 @@ def test_guided_prompts_check_defaults_public_resources_images_ecr_url( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.click.secho") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_images_no_image_uri( self, - patched_get_template_data, patched_signer_config_per_function, patched_click_secho, patched_sam_function_provider, - mock_get_template_function_resource_ids, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.gc._iac_stack.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.gc._iac_stack.get_overrideable_parameters.return_value = {} # Set ImageUri to be None, the sam app was never built. patched_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri=None)} ) - patched_get_template_artifacts_format.return_value = [IMAGE] patched_get_buildable_stacks.return_value = (Mock(), []) patched_prompt.side_effect = [ "sam-app", @@ -331,40 +390,37 @@ def test_guided_prompts_images_no_image_uri( patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() @patch("samcli.commands.deploy.guided_context.prompt") @patch("samcli.commands.deploy.guided_context.confirm") @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") - @patch("samcli.commands.deploy.guided_context.get_template_function_resource_ids") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.click.secho") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_images_blank_image_repository( self, - patched_get_template_data, patched_signer_config_per_function, patched_click_secho, patched_sam_function_provider, - mock_get_template_function_resource_ids, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = True + function_resource_mock = Mock() + function_resource_mock.item_id = "HelloWorldFunction" + self.gc._iac_stack.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.gc._iac_stack.get_overrideable_parameters.return_value = {} patched_sam_function_provider.return_value = MagicMock( functions={"HelloWorldFunction": MagicMock(packagetype=IMAGE, imageuri="mysamapp:v1")} ) - patched_get_template_artifacts_format.return_value = [IMAGE] patched_get_buildable_stacks.return_value = (Mock(), []) # set Image repository to be blank. patched_prompt.side_effect = [ @@ -378,7 +434,7 @@ def test_guided_prompts_images_blank_image_repository( patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) with self.assertRaises(GuidedDeployFailedError): - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() @parameterized.expand( [ @@ -399,30 +455,32 @@ def test_guided_prompts_images_blank_image_repository( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_with_given_capabilities( self, given_capabilities, - patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + # function_resource_mock = Mock() + # function_resource_mock.item_id = "HelloWorldFunction" + # self.gc._iac_stack.find_function_resources_of_package_type.return_value = [function_resource_mock] + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_signer_config_per_function.return_value = ({}, {}) patched_get_buildable_stacks.return_value = (Mock(), []) self.gc.capabilities = given_capabilities # Series of inputs to confirmations so that full range of questions are asked. patched_confirm.side_effect = [True, False, "", True] - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -445,32 +503,30 @@ def test_guided_prompts_with_given_capabilities( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_configuration_file_prompt_calls( self, - patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_get_buildable_stacks.return_value = (Mock(), []) patched_signer_config_per_function.return_value = ({}, {}) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_confirm.side_effect = [True, False, True, True, ""] patched_manage_stack.return_value = "managed_s3_stack" - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -505,34 +561,31 @@ def test_guided_prompts_check_configuration_file_prompt_calls( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_parameter_from_template( self, - patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} + patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_confirm.side_effect = [True, False, True, False, ""] patched_manage_stack.return_value = "managed_s3_stack" patched_signer_config_per_function.return_value = ({}, {}) - parameter_override_from_template = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} self.gc.parameter_overrides_from_cmdline = {} - self.gc.guided_prompts(parameter_override_keys=parameter_override_from_template) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -562,34 +615,31 @@ def test_guided_prompts_check_parameter_from_template( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_parameter_from_cmd_or_config( self, - patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, patched_confirm, patched_prompt, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} + patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] patched_confirm.side_effect = [True, False, True, False, ""] patched_signer_config_per_function.return_value = ({}, {}) patched_manage_stack.return_value = "managed_s3_stack" - parameter_override_from_template = {"MyTestKey": {"Default": "MyTemplateDefaultVal"}} self.gc.parameter_overrides_from_cmdline = {"MyTestKey": "OverridedValFromCmdLine", "NotUsedKey": "NotUsedVal"} - self.gc.guided_prompts(parameter_override_keys=parameter_override_from_template) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -630,16 +680,12 @@ def test_guided_prompts_check_parameter_from_cmd_or_config( @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_with_code_signing( self, given_sign_packages_flag, given_code_signing_configs, - patched_get_template_data, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_signer_config_per_function, patched_get_buildable_stacks, patchedauth_per_resource, @@ -650,14 +696,16 @@ def test_guided_prompts_with_code_signing( ): # given_sign_packages_flag = True # given_code_signing_configs = ({"MyFunction1"}, {"MyLayer1": {"MyFunction1"}, "MyLayer2": {"MyFunction1"}}) - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_signer_config_per_function.return_value = given_code_signing_configs patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patched_confirm.side_effect = [True, False, given_sign_packages_flag, "", True] - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), @@ -701,16 +749,12 @@ def test_guided_prompts_with_code_signing( @patch("samcli.commands.deploy.guided_context.manage_stack") @patch("samcli.commands.deploy.guided_context.auth_per_resource") @patch("samcli.commands.deploy.guided_context.SamLocalStackProvider.get_stacks") - @patch("samcli.commands.deploy.guided_context.get_template_artifacts_format") @patch("samcli.commands.deploy.guided_context.SamFunctionProvider") @patch("samcli.commands.deploy.guided_context.signer_config_per_function") - @patch("samcli.commands.deploy.guided_context.get_template_data") def test_guided_prompts_check_default_config_region( self, - patched_get_template_data, patched_signer_config_per_function, patched_sam_function_provider, - patched_get_template_artifacts_format, patched_get_buildable_stacks, patchedauth_per_resource, patched_manage_stack, @@ -718,9 +762,11 @@ def test_guided_prompts_check_default_config_region( patched_prompt, patched_get_session, ): - patched_get_template_data.return_value = {} + self.gc._iac_stack = MagicMock() + self.gc._iac_stack.has_assets_of_package_type.return_value = False + self.gc._iac_stack.get_overrideable_parameters.return_value = {} + patched_sam_function_provider.return_value = {} - patched_get_template_artifacts_format.return_value = [ZIP] patched_get_buildable_stacks.return_value = (Mock(), []) # Series of inputs to confirmations so that full range of questions are asked. patchedauth_per_resource.return_value = [("HelloWorldFunction", False)] @@ -730,7 +776,7 @@ def test_guided_prompts_check_default_config_region( patched_get_session.return_value.get_config_variable.return_value = "default_config_region" # setting the default region to None self.gc.region = None - self.gc.guided_prompts(parameter_override_keys=None) + self.gc.guided_prompts() # Now to check for all the defaults on confirmations. expected_confirmation_calls = [ call(f"\t{self.gc.start_bold}Confirm changes before deploy{self.gc.end_bold}", default=True), diff --git a/tests/unit/commands/package/test_command.py b/tests/unit/commands/package/test_command.py index 4d4f6593e7..21f625b089 100644 --- a/tests/unit/commands/package/test_command.py +++ b/tests/unit/commands/package/test_command.py @@ -22,6 +22,10 @@ def setUp(self): self.profile = None self.resolve_s3 = False self.signing_profiles = {"MyFunction": {"profile_name": "ProfileName", "profile_owner": "Profile Owner"}} + self.project_type = "CFN" + self.project = Mock() + self.iac = Mock() + self.stack_name = "" @patch("samcli.commands.package.command.click") @patch("samcli.commands.package.package_context.PackageContext") @@ -46,6 +50,10 @@ def test_all_args(self, package_command_context, click_mock): profile=self.profile, resolve_s3=self.resolve_s3, signing_profiles=self.signing_profiles, + project_type=self.project_type, + project=self.project, + iac=self.iac, + stack_name=self.stack_name, ) package_command_context.assert_called_with( @@ -63,6 +71,9 @@ def test_all_args(self, package_command_context, click_mock): region=self.region, profile=self.profile, signing_profiles=self.signing_profiles, + project=self.project, + iac=self.iac, + stack_name=self.stack_name, ) context_mock.run.assert_called_with() @@ -92,6 +103,10 @@ def test_all_args_resolve_s3(self, mock_managed_stack, package_command_context, profile=self.profile, resolve_s3=True, signing_profiles=self.signing_profiles, + project_type=self.project_type, + project=self.project, + iac=self.iac, + stack_name=self.stack_name, ) package_command_context.assert_called_with( @@ -109,6 +124,9 @@ def test_all_args_resolve_s3(self, mock_managed_stack, package_command_context, region=self.region, profile=self.profile, signing_profiles=self.signing_profiles, + project=self.project, + iac=self.iac, + stack_name=self.stack_name, ) context_mock.run.assert_called_with() diff --git a/tests/unit/commands/package/test_package_context.py b/tests/unit/commands/package/test_package_context.py index d25dafe778..e9ee37101f 100644 --- a/tests/unit/commands/package/test_package_context.py +++ b/tests/unit/commands/package/test_package_context.py @@ -1,6 +1,6 @@ """Test sam package command""" from unittest import TestCase -from unittest.mock import patch, MagicMock, Mock, call, ANY +from unittest.mock import MagicMixin, patch, MagicMock, Mock, call, ANY import tempfile @@ -25,11 +25,16 @@ def setUp(self): metadata={}, region=None, profile=None, + project=MagicMock(), + iac=MagicMock(), ) - @patch.object(Template, "export", MagicMock(sideeffect=OSError)) + @patch("samcli.commands.package.package_context.Template") @patch("boto3.Session") - def test_template_permissions_error(self, patched_boto): + @patch("json.dumps") + def test_template_permissions_error(self, json_dumps_mock, patched_boto, Template_mock): + template_instance = Template_mock.return_value + template_instance.export.side_effect = OSError with self.assertRaises(PackageFailedError): self.package_command_context.run() @@ -52,6 +57,8 @@ def test_template_path_valid_with_output_template(self, patched_boto): metadata={}, region=None, profile=None, + project=MagicMock(), + iac=MagicMock(), ) package_command_context.run() @@ -73,6 +80,8 @@ def test_template_path_valid(self, patched_boto): metadata={}, region=None, profile=None, + project=MagicMock(), + iac=MagicMock(), ) package_command_context.run() @@ -94,6 +103,8 @@ def test_template_path_valid_no_json(self, patched_boto): metadata={}, region=None, profile=None, + project=MagicMock(), + iac=MagicMock(), ) package_command_context.run() @@ -102,8 +113,7 @@ def test_template_path_valid_no_json(self, patched_boto): @patch("boto3.client") @patch("samcli.commands.package.package_context.get_boto_config_with_user_agent") def test_boto_clients_created_with_config(self, patched_get_config, patched_boto_client, patched_boto_session): - with self.assertRaises(PackageFailedError): - self.package_command_context.run() + self.package_command_context.run() patched_boto_client.assert_has_calls([call("s3", config=ANY)]) patched_boto_client.assert_has_calls([call("ecr", config=ANY)]) @@ -112,5 +122,3 @@ def test_boto_clients_created_with_config(self, patched_get_config, patched_boto patched_get_config.assert_has_calls( [call(region_name=ANY, signature_version=ANY), call(region_name=ANY), call(region_name=ANY)] ) - - print("hello") diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 491aad6a07..7e2fbb5393 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -13,7 +13,7 @@ from click.testing import CliRunner from unittest import TestCase -from unittest.mock import patch, ANY, Mock +from unittest.mock import patch, ANY, Mock, MagicMock import logging from samcli.lib.utils.packagetype import ZIP, IMAGE @@ -101,8 +101,8 @@ def test_validate(self, do_cli_mock): do_cli_mock.assert_called_with(ANY, str(Path(os.getcwd(), "mytemplate.yaml"))) @patch("samcli.commands.build.command.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_build(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_build(self, get_iac_plugin_mock, do_cli_mock): config_values = { "resource_logical_id": "foo", "template_file": "mytemplate.yaml", @@ -125,10 +125,9 @@ def test_build(self, CfnIacPlugin_mock, do_cli_mock): LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke(cli, []) @@ -163,8 +162,8 @@ def test_build(self, CfnIacPlugin_mock, do_cli_mock): ) @patch("samcli.commands.build.command.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_build_with_container_env_vars(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_build_with_container_env_vars(self, get_iac_plugin_mock, do_cli_mock): config_values = { "resource_logical_id": "foo", "template_file": "mytemplate.yaml", @@ -186,10 +185,9 @@ def test_build_with_container_env_vars(self, CfnIacPlugin_mock, do_cli_mock): from samcli.commands.build.command import cli LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke(cli, []) @@ -223,8 +221,8 @@ def test_build_with_container_env_vars(self, CfnIacPlugin_mock, do_cli_mock): ) @patch("samcli.commands.build.command.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_build_with_build_images(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_build_with_build_images(self, get_iac_plugin_mock, do_cli_mock): config_values = { "resource_logical_id": "foo", "template_file": "mytemplate.yaml", @@ -246,10 +244,9 @@ def test_build_with_build_images(self, CfnIacPlugin_mock, do_cli_mock): LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke(cli, []) @@ -284,8 +281,8 @@ def test_build_with_build_images(self, CfnIacPlugin_mock, do_cli_mock): ) @patch("samcli.commands.local.invoke.cli.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_local_invoke(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_local_invoke(self, get_iac_plugin_mock, do_cli_mock): config_values = { "function_logical_id": "foo", "template_file": "mytemplate.yaml", @@ -313,10 +310,9 @@ def test_local_invoke(self, CfnIacPlugin_mock, do_cli_mock): LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke(cli, []) @@ -352,8 +348,8 @@ def test_local_invoke(self, CfnIacPlugin_mock, do_cli_mock): ) @patch("samcli.commands.local.start_api.cli.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_local_start_api(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_local_start_api(self, get_iac_plugin_mock, do_cli_mock): config_values = { "template_file": "mytemplate.yaml", @@ -382,10 +378,9 @@ def test_local_start_api(self, CfnIacPlugin_mock, do_cli_mock): LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke(cli, []) @@ -423,8 +418,8 @@ def test_local_start_api(self, CfnIacPlugin_mock, do_cli_mock): ) @patch("samcli.commands.local.start_lambda.cli.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_local_start_lambda(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_local_start_lambda(self, get_iac_plugin_mock, do_cli_mock): config_values = { "template_file": "mytemplate.yaml", @@ -452,10 +447,9 @@ def test_local_start_lambda(self, CfnIacPlugin_mock, do_cli_mock): LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke(cli, []) @@ -490,20 +484,13 @@ def test_local_start_lambda(self, CfnIacPlugin_mock, do_cli_mock): project_mock, ) - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - @patch("samcli.commands._utils.options.get_template_artifacts_format") @patch("samcli.commands.package.command.do_cli") + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") def test_package( self, + get_iac_plugin_mock, do_cli_mock, - get_template_artifacts_format_mock, - cli_validation_artifacts_format_mock, - mock_get_template_function_resource_ids, ): - mock_get_template_function_resource_ids.return_value = ["HelloWorldFunction"] - cli_validation_artifacts_format_mock.return_value = [ZIP] - get_template_artifacts_format_mock.return_value = [ZIP] config_values = { "template_file": "mytemplate.yaml", "s3_bucket": "mybucket", @@ -522,6 +509,11 @@ def test_package( from samcli.commands.package.command import cli + iac_mock = MagicMock() + project_mock = MagicMock() + project_mock.stacks = [MagicMock()] + get_iac_plugin_mock.return_value = (iac_mock, project_mock) + LOG.debug(Path(config_path).read_text()) runner = CliRunner() result = runner.invoke(cli, []) @@ -548,12 +540,17 @@ def test_package( "myregion", None, False, + "CFN", + iac_mock, + project_mock, + None, ) @patch("samcli.commands._utils.options.get_template_artifacts_format") @patch("samcli.commands.package.command.do_cli") + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") def test_package_with_image_repository_and_image_repositories( - self, do_cli_mock, get_template_artifacts_format_mock + self, get_iac_plugin_mock, do_cli_mock, get_template_artifacts_format_mock ): get_template_artifacts_format_mock.return_value = [IMAGE] @@ -576,17 +573,20 @@ def test_package_with_image_repository_and_image_repositories( from samcli.commands.package.command import cli + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) + LOG.debug(Path(config_path).read_text()) runner = CliRunner() result = runner.invoke(cli, []) self.assertIsNotNone(result.exception) - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") @patch("samcli.commands.deploy.command.do_cli") - def test_deploy(self, do_cli_mock, get_template_artifacts_format_mock): + def test_deploy(self, do_cli_mock, get_iac_plugin_mock): - get_template_artifacts_format_mock.return_value = [ZIP] config_values = { "template_file": "mytemplate.yaml", "stack_name": "mystack", @@ -610,6 +610,10 @@ def test_deploy(self, do_cli_mock, get_template_artifacts_format_mock): "signing_profiles": "function=profile:owner", } + project_mock = MagicMock() + iac_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) + with samconfig_parameters(["deploy"], self.scratch_dir, **config_values) as config_path: from samcli.commands.deploy.command import cli @@ -651,10 +655,14 @@ def test_deploy(self, do_cli_mock, get_template_artifacts_format_mock): False, "samconfig.toml", "default", + "CFN", + iac_mock, + project_mock, ) + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") @patch("samcli.commands.deploy.command.do_cli") - def test_deploy_image_repositories_and_image_repository(self, do_cli_mock): + def test_deploy_image_repositories_and_image_repository(self, do_cli_mock, get_iac_plugin_mock): config_values = { "template_file": "mytemplate.yaml", @@ -684,16 +692,16 @@ def test_deploy_image_repositories_and_image_repository(self, do_cli_mock): from samcli.commands.deploy.command import cli + get_iac_plugin_mock.return_value = (Mock(), Mock()) + LOG.debug(Path(config_path).read_text()) runner = CliRunner() result = runner.invoke(cli, []) self.assertIsNotNone(result.exception) - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") @patch("samcli.commands.deploy.command.do_cli") - def test_deploy_different_parameter_override_format(self, do_cli_mock, get_template_artifacts_format_mock): - - get_template_artifacts_format_mock.return_value = [ZIP] + def test_deploy_different_parameter_override_format(self, do_cli_mock, get_iac_plugin_mock): config_values = { "template_file": "mytemplate.yaml", @@ -722,6 +730,10 @@ def test_deploy_different_parameter_override_format(self, do_cli_mock, get_templ from samcli.commands.deploy.command import cli + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) + LOG.debug(Path(config_path).read_text()) runner = CliRunner() result = runner.invoke(cli, []) @@ -759,6 +771,9 @@ def test_deploy_different_parameter_override_format(self, do_cli_mock, get_templ False, "samconfig.toml", "default", + "CFN", + iac_mock, + project_mock, ) @patch("samcli.commands.logs.command.do_cli") @@ -841,8 +856,8 @@ def tearDown(self): self.scratch_dir = None @patch("samcli.commands.local.start_lambda.cli.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_override_with_cli_params(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_override_with_cli_params(self, get_iac_plugin_mock, do_cli_mock): config_values = { "template_file": "mytemplate.yaml", @@ -869,10 +884,9 @@ def test_override_with_cli_params(self, CfnIacPlugin_mock, do_cli_mock): from samcli.commands.local.start_lambda.cli import cli LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke( @@ -945,8 +959,8 @@ def test_override_with_cli_params(self, CfnIacPlugin_mock, do_cli_mock): ) @patch("samcli.commands.local.start_lambda.cli.do_cli") - @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") - def test_override_with_cli_params_and_envvars(self, CfnIacPlugin_mock, do_cli_mock): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_override_with_cli_params_and_envvars(self, get_iac_plugin_mock, do_cli_mock): config_values = { "template_file": "mytemplate.yaml", @@ -973,10 +987,9 @@ def test_override_with_cli_params_and_envvars(self, CfnIacPlugin_mock, do_cli_mo LOG.debug(Path(config_path).read_text()) - iac_mock = Mock() - project_mock = Mock() - CfnIacPlugin_mock.return_value = iac_mock - iac_mock.get_project.return_value = project_mock + iac_mock = MagicMock() + project_mock = MagicMock() + get_iac_plugin_mock.return_value = (iac_mock, project_mock) runner = CliRunner() result = runner.invoke( diff --git a/tests/unit/lib/cli_validation/test_image_repository_validation.py b/tests/unit/lib/cli_validation/test_image_repository_validation.py index 9773cbc9d0..9933909427 100644 --- a/tests/unit/lib/cli_validation/test_image_repository_validation.py +++ b/tests/unit/lib/cli_validation/test_image_repository_validation.py @@ -4,146 +4,194 @@ import click from samcli.lib.cli_validation.image_repository_validation import image_repository_validation -from samcli.lib.utils.packagetype import ZIP, IMAGE + + +def _make_ctx_params_side_effect_func(params): + def side_effect(key, default=None): + return params.get(key, default) + + return side_effect class TestImageRepositoryValidation(TestCase): def setUp(self): @image_repository_validation - def foo(): + def foo(*args, **kwargs): pass self.foobar = foo @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_success_ZIP(self, mock_artifacts, mock_resource_ids, mock_click): - mock_artifacts.return_value = [ZIP] - mock_resource_ids.return_value = ["HelloWorldFunction"] + def test_image_repository_validation_success_ZIP(self, mock_click): + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = False + + params = { + "guided": False, + "image_repository": False, + "image_repositories": False, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [False, False, False, False, False, MagicMock()] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context - self.foobar() + self.foobar(project=project_mock) @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_success_IMAGE_image_repository( - self, mock_artifacts, mock_resource_ids, mock_click - ): - mock_artifacts.return_value = [IMAGE] - mock_resource_ids.return_value = ["HelloWorldFunction"] + def test_image_repository_validation_success_IMAGE_image_repository(self, mock_click): + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = True + function_mock = MagicMock() + function_mock.item_id.return_value = "HelloWorldFunction" + stack_mock.find_function_resources_of_package_type.return_value = [function_mock] + + params = { + "guided": False, + "image_repository": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1", + "image_repositories": False, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [ - False, - False, - "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1", - False, - False, - MagicMock(), - ] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context - self.foobar() + self.foobar(project=project_mock) @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_success_IMAGE_image_repositories( - self, mock_artifacts, mock_resource_ids, mock_click - ): - mock_artifacts.return_value = [IMAGE] - mock_resource_ids.return_value = ["HelloWorldFunction"] + def test_image_repository_validation_success_IMAGE_image_repositories(self, mock_click): + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = True + function_mock = MagicMock() + function_mock.item_id = "HelloWorldFunction" + stack_mock.find_function_resources_of_package_type.return_value = [function_mock] + + params = { + "guided": False, + "image_repository": False, + "image_repositories": {"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [ - False, - False, - False, - {"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, - False, - MagicMock(), - ] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context - self.foobar() + + self.foobar(project=project_mock) @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_failure_IMAGE_image_repositories_and_image_repository( - self, mock_artifacts, mock_resource_ids, mock_click - ): + def test_image_repository_validation_failure_IMAGE_image_repositories_and_image_repository(self, mock_click): + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = True + function_mock = MagicMock() + function_mock.item_id = "HelloWorldFunction" + stack_mock.find_function_resources_of_package_type.return_value = [function_mock] mock_click.BadOptionUsage = click.BadOptionUsage - mock_artifacts.return_value = [IMAGE] - mock_resource_ids.return_value = ["HelloWorldFunction"] + + params = { + "guided": False, + "image_repository": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1", + "image_repositories": {"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [ - False, - False, - "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1", - {"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, - False, - MagicMock(), - ] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context with self.assertRaises(click.BadOptionUsage) as ex: - self.foobar() + self.foobar(project=project_mock) self.assertIn("'--image-repositories' and '--image-repository' cannot be provided", ex.exception.message) @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_failure_IMAGE_image_repositories_incomplete( - self, mock_artifacts, mock_resource_ids, mock_click - ): + def test_image_repository_validation_failure_IMAGE_image_repositories_incomplete(self, mock_click): + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = True + function_mock = MagicMock() + function_mock.item_id = "HelloWorldFunction" + function2_mock = MagicMock() + function2_mock.item_id = "HelloWorldFunction2" + stack_mock.find_function_resources_of_package_type.return_value = [function_mock, function2_mock] mock_click.BadOptionUsage = click.BadOptionUsage - mock_artifacts.return_value = [IMAGE] - mock_resource_ids.return_value = ["HelloWorldFunction", "HelloWorldFunction2"] + + params = { + "guided": False, + "image_repository": False, + "image_repositories": {"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [ - False, - False, - False, - {"HelloWorldFunction": "123456789012.dkr.ecr.us-east-1.amazonaws.com/test1"}, - False, - MagicMock(), - ] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context with self.assertRaises(click.BadOptionUsage) as ex: - self.foobar() + self.foobar(project=project_mock) self.assertIn("Incomplete list of function logical ids specified", ex.exception.message) @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_failure_IMAGE_missing_image_repositories( - self, mock_artifacts, mock_resource_ids, mock_click - ): + def test_image_repository_validation_failure_IMAGE_missing_image_repositories(self, mock_click): + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = True + function_mock = MagicMock() + function_mock.item_id = "HelloWorldFunction" + function2_mock = MagicMock() + function2_mock.item_id = "HelloWorldFunction2" + stack_mock.find_function_resources_of_package_type.return_value = [function_mock, function2_mock] mock_click.BadOptionUsage = click.BadOptionUsage - mock_artifacts.return_value = [IMAGE] - mock_resource_ids.return_value = ["HelloWorldFunction", "HelloWorldFunction2"] + + params = { + "guided": False, + "image_repository": False, + "image_repositories": None, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [False, False, False, None, False, MagicMock()] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context with self.assertRaises(click.BadOptionUsage) as ex: - self.foobar() + self.foobar(project=project_mock) self.assertIn("Missing option '--image-repository' or '--image-repositories'", ex.exception.message) @patch("samcli.lib.cli_validation.image_repository_validation.click") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_function_resource_ids") - @patch("samcli.lib.cli_validation.image_repository_validation.get_template_artifacts_format") - def test_image_repository_validation_success_missing_image_repositories_guided( - self, mock_artifacts, mock_resource_ids, mock_click - ): + def test_image_repository_validation_success_missing_image_repositories_guided(self, mock_click): # Guided allows for filling of the image repository values. + project_mock = MagicMock() + stack_mock = MagicMock() + project_mock.stacks = [stack_mock] + stack_mock.has_assets_of_package_type.return_value = True + function_mock = MagicMock() + function_mock.item_id = "HelloWorldFunction" + function2_mock = MagicMock() + function2_mock.item_id = "HelloWorldFunction2" + stack_mock.find_function_resources_of_package_type.return_value = [function_mock, function2_mock] mock_click.BadOptionUsage = click.BadOptionUsage - mock_artifacts.return_value = [IMAGE] - mock_resource_ids.return_value = ["HelloWorldFunction", "HelloWorldFunction2"] + + params = { + "guided": True, + "image_repository": False, + "image_repositories": None, + "project_type": "CFN", + "stack_name": None, + } mock_context = MagicMock() - mock_context.params.get.side_effect = [True, True, False, None, False, MagicMock()] + mock_context.params.get.side_effect = _make_ctx_params_side_effect_func(params) mock_click.get_current_context.return_value = mock_context - self.foobar() + + self.foobar(project=project_mock) diff --git a/tests/unit/lib/iac/cdk/test_cloud_assembly.py b/tests/unit/lib/iac/cdk/test_cloud_assembly.py index 39cd603fea..2cc9d1709f 100644 --- a/tests/unit/lib/iac/cdk/test_cloud_assembly.py +++ b/tests/unit/lib/iac/cdk/test_cloud_assembly.py @@ -110,6 +110,11 @@ def test_find_asset_by_id(self): } self.assertEqual(asset, expected) + def test_find_metadata_by_type(self): + stack = self.cloud_assembly.find_stack_by_stack_name("root-stack") + metadata = stack.find_metadata_by_type("aws:cdk:asset") + self.assertEqual(len(metadata), 5) + def test_find_asset_by_path(self): stack = self.cloud_assembly.find_stack_by_stack_name("root-stack") asset = stack.find_asset_by_path("asset.97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0e") @@ -207,6 +212,16 @@ def test_find_node_by_path(self): self.assertEqual(node.id, "Resource") self.assertEqual(node.path, "root-stack/container-function/Resource") + def test_find_node_by_path_abs_path(self): + node = self.tree.find_node_by_path("/root-stack/container-function/Resource") + self.assertIsInstance(node, CloudAssemblyTreeNode) + self.assertEqual(node.id, "Resource") + self.assertEqual(node.path, "root-stack/container-function/Resource") + + def test_find_node_by_path_not_found(self): + node = self.tree.find_node_by_path("root-stack/faked-resource") + self.assertIsNone(node) + class TestCloudAssemblyTreeNode(TestCase): def setUp(self) -> None: diff --git a/tests/unit/lib/iac/cdk/test_data/cdk.out/root-stack.template.normalized.json b/tests/unit/lib/iac/cdk/test_data/cdk.out/root-stack.template.normalized.json index 751b2b3f0b..d478b28be8 100644 --- a/tests/unit/lib/iac/cdk/test_data/cdk.out/root-stack.template.normalized.json +++ b/tests/unit/lib/iac/cdk/test_data/cdk.out/root-stack.template.normalized.json @@ -80,7 +80,7 @@ "Metadata": { "aws:cdk:path": "root-stack/container-function/Resource", "aws:asset:path": "asset.8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e", - "aws:asset:property": "Code.ImageUri" + "aws:asset:property": "Code" } }, "remotenestedstack": { diff --git a/tests/unit/lib/iac/cdk/test_plugin.py b/tests/unit/lib/iac/cdk/test_plugin.py index 6108d53bf4..23be1543da 100644 --- a/tests/unit/lib/iac/cdk/test_plugin.py +++ b/tests/unit/lib/iac/cdk/test_plugin.py @@ -1,12 +1,42 @@ import os +from samcli.commands._utils.template import TemplateFormat import shutil +import subprocess from pathlib import Path +from typing import Dict, Mapping -from samcli.lib.iac.interface import Project, LookupPath, LookupPathType from unittest import TestCase -from unittest.mock import Mock +from unittest.mock import ANY, Mock, mock_open, patch -from samcli.lib.iac.cdk.plugin import CdkPlugin +from samcli.lib.iac.interface import ( + DictSection, + Project, + LookupPath, + LookupPathType, + Resource, + S3Asset, + ImageAsset, + SimpleSection, + Stack, +) +from samcli.lib.iac.cdk.plugin import ( + CdkPlugin, + _collect_assets, + _write_stack, + _shallow_clone_asset, + _undo_normalize_resource_metadata, + _collect_stack_assets, + _update_built_artifacts, + _collect_project_assets, + _update_asset_params_default_values, + _get_cdk_executable_path, +) +from samcli.lib.iac.cdk.exceptions import CdkToolkitNotInstalledError, InvalidCloudAssemblyError +from samcli.lib.iac.cdk.constants import ( + MANIFEST_FILENAME, + TREE_FILENAME, + OUT_FILENAME, +) from tests.unit.lib.iac.cdk.helper import read_json_file @@ -15,39 +45,679 @@ class TestCdkPlugin(TestCase): def setUp(self) -> None: - context = Mock() - context.command_params = {"cdk_app": CLOUD_ASSEMBLY_DIR} - self.plugin = CdkPlugin(context) - self.project = self.plugin.get_project([LookupPath(os.path.dirname(__file__), LookupPathType.SOURCE)]) - - def test_get_project(self): - self.assertIsInstance(self.project, Project) - self.assertEqual(len(self.project.stacks), 2) - root_stack = self.project.stacks[0] - self.assertEqual(len(root_stack.assets), 5) - - def test_write_project(self): - filenames = [ - "root-stack.template.json", - "rootstacknestedstackACD02B51.nested.template.json", - "rootstacknestedstacknestednestedstackE7ADAD2C.nested.template.json", - "Stack2.template.json", + self.command_params = {"cdk_app": CLOUD_ASSEMBLY_DIR} + # self.cdk_synth_mock = + # self.plugin = CdkPlugin(context) + # self.project = self.plugin.get_project([LookupPath(os.path.dirname(__file__), LookupPathType.SOURCE)]) + + @patch("os.path.isfile", return_value=True) + @patch("os.path.abspath", return_value="/path/to/cloud_assemble") + @patch("os.path.exists", return_value=True) + def test_get_project_build_type_lookup_path(self, path_exists_mock, abspath_mock, is_file_mock): + cdk_plugin = CdkPlugin(self.command_params) + cdk_plugin._get_project_from_cloud_assembly = Mock() + cdk_plugin._get_project_from_cloud_assembly.return_value = Project(stacks=[Mock(), Mock()]) + + project = cdk_plugin.get_project([LookupPath("lookup/path", LookupPathType.BUILD)]) + cdk_plugin._get_project_from_cloud_assembly.assert_called_once_with(abspath_mock.return_value) + self.assertIsInstance(project, Project) + self.assertEqual(len(project.stacks), 2) + + @patch("os.path.isfile", return_value=True) + @patch("os.path.abspath", return_value="/path/to/cloud_assemble") + @patch("os.path.exists", return_value=True) + def test_get_project_source_type_lookup_path(self, path_exists_mock, abspath_mock, is_file_mock): + cdk_plugin = CdkPlugin(self.command_params) + cdk_plugin._get_project_from_cloud_assembly = Mock() + cdk_plugin._get_project_from_cloud_assembly.return_value = Project(stacks=[Mock(), Mock()]) + cdk_plugin._cdk_synth = Mock() + cdk_plugin._cdk_synth.return_value = abspath_mock.return_value + + project = cdk_plugin.get_project([LookupPath("lookup/path", LookupPathType.SOURCE)]) + cdk_plugin._cdk_synth.assert_called_once_with(app=CLOUD_ASSEMBLY_DIR, context=None) + cdk_plugin._get_project_from_cloud_assembly.assert_called_once_with(abspath_mock.return_value) + self.assertIsInstance(project, Project) + self.assertEqual(len(project.stacks), 2) + + @patch("os.path.isfile", return_value=True) + def test_get_project_invalid_lookup_path(self, is_file_mock): + cdk_plugin = CdkPlugin(self.command_params) + + with self.assertRaises(InvalidCloudAssemblyError): + cdk_plugin.get_project([]) + + @patch("shutil.copy2") + @patch("samcli.lib.iac.cdk.plugin._update_built_artifacts") + @patch("samcli.lib.iac.cdk.plugin._write_stack") + def test_write_project(self, write_stack_mock, update_built_artifacts_mock, copy2_mock): + cdk_plugin = CdkPlugin(self.command_params) + project_mock = Mock() + project_mock.stacks = [Mock()] + build_dir = "build_dir" + cdk_plugin.write_project(project_mock, build_dir) + + self.assertEqual(copy2_mock.call_count, 3) + + update_built_artifacts_mock.assert_called_once_with(project_mock, cdk_plugin._cloud_assembly_dir, build_dir) + for stack in project_mock.stacks: + write_stack_mock.assert_called_once_with(stack, cdk_plugin._cloud_assembly_dir, build_dir) + + def test_should_update_property_after_package(self): + asset = ImageAsset() + cdk_plugin = CdkPlugin(self.command_params) + self.assertTrue(cdk_plugin.should_update_property_after_package(asset)) + + def test_should_not_update_property_after_package(self): + asset = S3Asset() + cdk_plugin = CdkPlugin(self.command_params) + self.assertFalse(cdk_plugin.should_update_property_after_package(asset)) + + @patch("samcli.lib.iac.cdk.plugin._update_asset_params_default_values") + def test_update_asset_params_default_values_after_packaging(self, update_asset_params_default_values_mock): + parameters_mock = {} + + def param_side_effect_1(asset, params): + parameters_mock["foo"] = "bar" + + def param_side_efftec_2(stack, params): + parameters_mock["bax"] = "baz" + + update_asset_params_default_values_mock.side_effect = param_side_effect_1 + + stack_mock = Mock() + resource_mock = Mock() + stack_mock.get.return_value = {"resource": resource_mock} + asset = S3Asset() + asset.extra_details = {"assetParameters": {"param": "param"}} + nested_stack_mock = Mock() + resource_mock.nested_stack = nested_stack_mock + resource_mock.assets = [asset] + cdk_plugin = CdkPlugin(self.command_params) + original_func = cdk_plugin.update_asset_params_default_values_after_packaging + cdk_plugin.update_asset_params_default_values_after_packaging = Mock() + cdk_plugin.update_asset_params_default_values_after_packaging.side_effect = param_side_efftec_2 + + original_func(stack_mock, parameters_mock) + update_asset_params_default_values_mock.assert_called_once_with(asset, parameters_mock) + cdk_plugin.update_asset_params_default_values_after_packaging.assert_called_once_with( + nested_stack_mock, parameters_mock + ) + + self.assertEqual( + parameters_mock, + { + "foo": "bar", + "bax": "baz", + }, + ) + + @patch("samcli.lib.iac.cdk.plugin._undo_normalize_resource_metadata") + def test_update_resource_after_packaging(self, undo_normalize_resource_metadata_mock): + resource_mock = Mock() + asset_mock = S3Asset() + asset_mock.extra_details = {"assetParameters": {"foo": "bar"}} + resource_mock.assets = [asset_mock] + + cdk_plugin = CdkPlugin(self.command_params) + cdk_plugin.update_resource_after_packaging(resource_mock) + undo_normalize_resource_metadata_mock.assert_called_once_with(resource_mock) + + @patch("subprocess.check_output") + @patch("os.path.isdir", return_value=False) + @patch("samcli.lib.iac.cdk.plugin.copy_tree") + @patch("samcli.lib.iac.cdk.plugin._get_cdk_executable_path", return_value="cdk") + def test_cdk_synth_app_is_executable_with_context( + self, get_cdk_executable_path_mock, copy_tree_mock, isdir_mock, check_output_mock + ): + cdk_plugin = CdkPlugin(self.command_params) + app = "cdk_app_executable" + context = ["key1=value1", "key2=value2"] + cloud_assembly_dir = cdk_plugin._cdk_synth(app, context) + check_output_mock.assert_called_once_with( + [ + "cdk", + "synth", + "--no-staging", + "--app", + app, + "-o", + cdk_plugin._cloud_assembly_dir, + "--context", + context[0], + "--context", + context[1], + ], + stderr=subprocess.STDOUT, + ) + copy_tree_mock.assert_not_called() + self.assertEqual(cloud_assembly_dir, cdk_plugin._cloud_assembly_dir) + + @patch("subprocess.check_output") + @patch("os.path.isdir", return_value=True) + @patch("samcli.lib.iac.cdk.plugin.copy_tree") + @patch("samcli.lib.iac.cdk.plugin._get_cdk_executable_path", return_value="cdk") + def test_cdk_synth_app_is_dir(self, get_cdk_executable_path_mock, copy_tree_mock, isdir_mock, check_output_mock): + cdk_plugin = CdkPlugin(self.command_params) + app = "path/to/cloud_assembly" + cloud_assembly_dir = cdk_plugin._cdk_synth(app) + check_output_mock.assert_called_once_with( + [ + "cdk", + "synth", + "--no-staging", + "--app", + app, + ], + stderr=subprocess.STDOUT, + ) + copy_tree_mock.assert_called_once_with(app, cdk_plugin._cloud_assembly_dir) + self.assertEqual(cloud_assembly_dir, cdk_plugin._cloud_assembly_dir) + + @patch("subprocess.check_output") + @patch("os.path.isdir", return_value=False) + @patch("samcli.lib.iac.cdk.plugin.copy_tree") + @patch("samcli.lib.iac.cdk.plugin._get_cdk_executable_path", return_value="cdk") + def test_cdk_synth_app_is_none(self, get_cdk_executable_path_mock, copy_tree_mock, isdir_mock, check_output_mock): + cdk_plugin = CdkPlugin(self.command_params) + app = None + cloud_assembly_dir = cdk_plugin._cdk_synth(app) + check_output_mock.assert_called_once_with( + [ + "cdk", + "synth", + "--no-staging", + "-o", + cdk_plugin._cloud_assembly_dir, + ], + stderr=subprocess.STDOUT, + ) + copy_tree_mock.assert_not_called() + self.assertEqual(cloud_assembly_dir, cdk_plugin._cloud_assembly_dir) + + @patch("samcli.lib.iac.cdk.plugin.CloudAssembly") + def test_get_project_from_cloud_assembly(self, cloud_assembly_class_mock): + cloud_assembly_mock = Mock() + cloud_assembly_mock.stacks = [Mock(), Mock()] + cloud_assembly_class_mock.return_value = cloud_assembly_mock + cdk_plugin = CdkPlugin(self.command_params) + build_stack_mock = Mock() + stack_mock1 = Mock() + stack_mock2 = Mock() + build_stack_mock.side_effect = [stack_mock1, stack_mock2] + cdk_plugin._build_stack = build_stack_mock + cloud_assembly_path = "path/to/cloud_assembly" + project = cdk_plugin._get_project_from_cloud_assembly(cloud_assembly_path) + cloud_assembly_class_mock.assert_called_once_with(cloud_assembly_path, cdk_plugin._source_dir) + self.assertEqual(cdk_plugin._build_stack.call_count, 2) + self.assertEqual(len(project.stacks), 2) + + @patch("samcli.lib.iac.cdk.plugin._collect_assets") + def test_build_stack(self, collect_assets_mock): + cloud_assembly_mock = Mock() + ca_stack_mock = Mock() + ca_stack_mock.template = { + "Resources": {}, + "Parameters": { + "Param1": {"Type": "String", "Default": "foo"}, + }, + "OtherMapping": {}, + "OtherKey": "other", + } + ca_stack_mock.stack_name = "test_stack" + ca_stack_mock.template_file = "to/template" + ca_stack_mock.template_full_path = "/path/to/template" + s3_asset = S3Asset() + image_asset = ImageAsset() + assets = {"s3/asset": s3_asset, "image/asset": image_asset} + collect_assets_mock.return_value = assets + cdk_plugin = CdkPlugin(self.command_params) + cdk_plugin._build_resources_section = Mock() + stack = cdk_plugin._build_stack(cloud_assembly_mock, ca_stack_mock) + cdk_plugin._build_resources_section.assert_called_once_with(assets, ca_stack_mock, cloud_assembly_mock, ANY, {}) + self.assertIsInstance(stack, Stack) + self.assertEqual(stack.stack_id, "test_stack") + self.assertEqual(stack.name, "test_stack") + self.assertFalse(stack.is_nested, False) + self.assertEqual(list(stack.sections.keys()), list(ca_stack_mock.template.keys())) + self.assertIn("Param1", stack["Parameters"]) + + def test_build_resource_section_image_asset(self): + image_asset = ImageAsset() + assets = { + "/path/to/asset": image_asset, + } + dict_section = DictSection("Resources") + section_dict = { + "logical_id": { + "Type": "Resource", + "Properties": { + "some_prop": "value", + }, + "Metadata": { + "aws:cdk:path": "stack/resouce", + "aws:asset:path": "/path/to/asset", + "aws:asset:property": "some_prop", + "aws:asset:local_image": "image:tag", + }, + }, + } + ca_stack_mock = Mock() + ca_stack_mock.find_nested_stack_by_logical_id.return_value = None + cloud_assembly_mock = Mock() + node_mock = Mock() + node_mock.id = "resource_id" + node_mock.is_l2_construct_resource.return_value = False + cloud_assembly_mock.tree.find_node_by_path.return_value = node_mock + + cdk_plugin = CdkPlugin(self.command_params) + cdk_plugin._build_resources_section(assets, ca_stack_mock, cloud_assembly_mock, dict_section, section_dict) + self.assertTrue("logical_id" in dict_section) + resource = dict_section["logical_id"] + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.key, "logical_id") + self.assertEqual(resource.body, section_dict["logical_id"]) + self.assertEqual(resource.assets, [image_asset]) + self.assertEqual(resource["Properties"]["some_prop"], "image:tag") + self.assertEqual(resource.item_id, "resource_id") + + def test_build_resource_section_nested_stack(self): + nested_stack_asset = S3Asset() + assets = {"/path/to/asset": nested_stack_asset} + dict_section = DictSection("Resources") + section_dict = { + "logical_id": { + "Type": "AWS::CloudFormation::Stack", + "Properties": { + "some_prop": "value", + }, + "Metadata": { + "aws:cdk:path": "stack/resouce", + "aws:asset:path": "/path/to/asset", + "aws:asset:property": "some_prop", + }, + }, + } + ca_stack_mock = Mock() + ca_stack_mock.find_nested_stack_by_logical_id.return_value = ca_nested_stack_mock = Mock() + cloud_assembly_mock = Mock() + node_mock = Mock() + node_mock.id = "resource_id" + node_mock.is_l2_construct_resource.return_value = False + cloud_assembly_mock.tree.find_node_by_path.return_value = node_mock + + cdk_plugin = CdkPlugin(self.command_params) + cdk_plugin._build_stack = Mock() + cdk_plugin._build_stack.return_value = Mock() + cdk_plugin._build_resources_section(assets, ca_stack_mock, cloud_assembly_mock, dict_section, section_dict) + self.assertTrue("logical_id" in dict_section) + resource = dict_section["logical_id"] + self.assertIsInstance(resource, Resource) + self.assertEqual(resource.key, "logical_id") + self.assertEqual(resource.body, section_dict["logical_id"]) + self.assertEqual(resource.assets, [nested_stack_asset]) + self.assertEqual(resource.item_id, "resource_id") + self.assertEqual(resource["Properties"]["some_prop"], "/path/to/asset") + + cdk_plugin._build_stack.assert_called_once_with(cloud_assembly_mock, ca_nested_stack_mock) + + +class TestCollectAssets(TestCase): + @patch("os.path.normpath", return_value="/path/to/asset") + @patch("os.path.join", return_value="joined/path") + def test_collect_assets(self, normpath_mock, join_mock): + ca_assets = [ + { + "repositoryName": "aws-cdk/assets", + "imageTag": "8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e", + "id": "8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e", + "packaging": "container-image", + "path": "asset.8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e", + "sourceHash": "8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e", + }, + { + "path": "asset.97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0e", + "id": "97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0e", + "packaging": "zip", + "sourceHash": "97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0e", + "s3BucketParameter": "AssetParameters97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0eS3BucketDD3A7E9A", + "s3KeyParameter": "AssetParameters97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0eS3VersionKeyCEC49660", + "artifactHashParameter": "AssetParameters97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0eArtifactHashBD7218A6", + }, ] - build_dir = Path(CLOUD_ASSEMBLY_DIR, ".build") - build_dir_path = str(build_dir) - build_dir.mkdir(exist_ok=True) - self.plugin.write_project(self.project, build_dir_path) - self.assertTrue(os.path.isfile(os.path.join(build_dir_path, "manifest.json"))) - self.assertTrue(os.path.isfile(os.path.join(build_dir_path, "tree.json"))) - self.assertTrue(os.path.isfile(os.path.join(build_dir_path, "cdk.out"))) - for filename in filenames: - self.assertTrue(os.path.isfile(os.path.join(build_dir_path, filename))) - expected = read_json_file( - os.path.join(CLOUD_ASSEMBLY_DIR, filename.replace(".template.json", ".template.normalized.json")), - "/", - f"{str(CLOUD_ASSEMBLY_DIR)}{os.path.sep}", - ) - actual = read_json_file(os.path.join(build_dir_path, filename)) - self.assertEqual(actual, expected) - - shutil.rmtree(build_dir) + ca_stack_mock = Mock() + ca_stack_mock.assets = ca_assets + assets = _collect_assets(ca_stack_mock) + self.assertIn("asset.8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e", assets) + self.assertIsInstance( + assets["asset.8db62da8c9a661a051169ec5710e7c48e471f6eb556b9d853c77e0efd9689d7e"], ImageAsset + ) + self.assertIn("asset.97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0e", assets) + self.assertIsInstance(assets["asset.97bd9d4f97345f890d541646ecd5ec0348b62d5618cedfd76a6f6dc94f2e4f0e"], S3Asset) + + +class TestWriteStack(TestCase): + def setUp(self): + self.original_func = _write_stack + + @patch("os.path.join", side_effect=["src_template_path", "stack_build_location", "build_dir/nested_stack"]) + @patch("samcli.lib.iac.cdk.plugin._undo_normalize_resource_metadata") + @patch("samcli.lib.iac.cdk.plugin.move_template") + def test_write_stack_s3_asset(self, move_template_mock, undo_normalize_mock, path_join_mock): + stack = Stack(stack_id="test_stack") + stack["Resources"] = DictSection("Resources") + stack.extra_details["template_file"] = "template_file" + resource = Resource( + "Function1", + body={ + "Type": "AWS::Lambda::Function", + "Properties": {}, + "Metadata": { + "aws:cdk:path": "stack/resouce", + "aws:asset:path": "source_asset", + "aws:asset:property": "Code", + }, + }, + ) + stack["Resources"]["Function1"] = resource + s3_asset = S3Asset( + asset_id="id", + updated_source_path="updated_source_path", + ) + resource.assets.append(s3_asset) + + _write_stack(stack, "cloud_assembly_dir", "build_dir") + undo_normalize_mock.assert_called_once_with(resource) + self.assertEqual(resource["Metadata"]["aws:asset:path"], "updated_source_path") + move_template_mock.assert_called_once_with( + "src_template_path", "stack_build_location", stack, output_format=TemplateFormat.JSON + ) + + @patch("os.path.join", side_effect=["src_template_path", "stack_build_location", "build_dir/nested_stack"]) + @patch("samcli.lib.iac.cdk.plugin._undo_normalize_resource_metadata") + @patch("samcli.lib.iac.cdk.plugin.move_template") + def test_write_stack_image_asset(self, move_template_mock, undo_normalize_mock, path_join_mock): + stack = Stack(stack_id="test_stack") + stack["Resources"] = DictSection("Resources") + stack.extra_details["template_file"] = "template_file" + resource = Resource( + "Function1", + body={ + "Type": "AWS::Lambda::Function", + "Properties": {}, + "Metadata": { + "aws:cdk:path": "stack/resouce", + "aws:asset:path": "source_asset", + "aws:asset:property": "Code", + }, + }, + ) + stack["Resources"]["Function1"] = resource + image_asset = ImageAsset( + asset_id="id", + source_local_image="image:tag", + ) + resource.assets.append(image_asset) + + _write_stack(stack, "cloud_assembly_dir", "build_dir") + undo_normalize_mock.assert_called_once_with(resource) + self.assertIn("aws:asset:local_image", resource["Metadata"]) + self.assertEqual(resource["Metadata"]["aws:asset:local_image"], "image:tag") + move_template_mock.assert_called_once_with( + "src_template_path", "stack_build_location", stack, output_format=TemplateFormat.JSON + ) + + @patch("os.path.join", side_effect=["src_template_path", "stack_build_location", "build_dir/nested_stack"]) + @patch("samcli.lib.iac.cdk.plugin._undo_normalize_resource_metadata") + @patch("samcli.lib.iac.cdk.plugin.move_template") + @patch("samcli.lib.iac.cdk.plugin._write_stack") + def test_write_stack_s3_asset_nested_stack( + self, write_stack_mock, move_template_mock, undo_normalize_mock, path_join_mock + ): + stack = Stack(stack_id="test_stack") + stack["Resources"] = DictSection("Resources") + stack.extra_details["template_file"] = "template_file" + resource = Resource( + "NestedStack", + body={ + "Type": "AWS::Lambda::Function", + "Properties": {}, + "Metadata": { + "aws:cdk:path": "stack/resouce", + "aws:asset:path": "source_asset", + "aws:asset:property": "TemplateURL", + }, + }, + ) + stack["Resources"]["Function1"] = resource + s3_asset = S3Asset( + asset_id="id", + updated_source_path="updated_source_path", + source_property="TemplateURL", + ) + resource.assets.append(s3_asset) + resource.nested_stack = Stack( + sections={"Resources": {}}, extra_details={"template_file": "hello.nested-stack.json"} + ) + + self.original_func(stack, "cloud_assembly_dir", "build_dir") + undo_normalize_mock.assert_called_once_with(resource) + self.assertEqual(resource["Metadata"]["aws:asset:path"], "build_dir/nested_stack") + self.assertEqual(s3_asset.updated_source_path, "build_dir/nested_stack") + write_stack_mock.assert_called_once_with(resource.nested_stack, "cloud_assembly_dir", "build_dir") + move_template_mock.assert_called_once_with( + "src_template_path", "stack_build_location", stack, output_format=TemplateFormat.JSON + ) + + +class TestUndoNormalizeResourceMetadata(TestCase): + def test_undo_normalize_resource_metadata(self): + resource = Resource("Function") + resource["Key"] = "NewVal" + resource.extra_details["original_body"] = {"Key": "OriginalVal"} + _undo_normalize_resource_metadata(resource) + self.assertEqual(resource["Key"], "OriginalVal") + + +class TestCollectStackAssets(TestCase): + def setUp(self): + self.original_func = _collect_stack_assets + + def test_collect_stack_assets(self): + s3_asset = S3Asset(asset_id="s3") + image_asset = ImageAsset(asset_id="image") + stack = Stack() + dict_section = DictSection() + dict_section["Function1"] = Resource(assets=[s3_asset]) + dict_section["Function2"] = Resource(assets=[image_asset]) + stack.sections["Resources"] = dict_section + + collected = _collect_stack_assets(stack) + self.assertIn("s3", collected) + self.assertIn("image", collected) + self.assertEqual(collected["s3"], s3_asset) + self.assertEqual(collected["image"], image_asset) + + @patch("samcli.lib.iac.cdk.plugin._collect_stack_assets") + def test_collect_stack_assets_nested_stack(self, collect_stack_assets_mock): + nested_stack_asset = S3Asset(asset_id="nested_stack") + stack = Stack() + dict_section = DictSection() + nested_stack = Stack(sections={"Resources": {}}) + dict_section["NestedStack"] = Resource(assets=[nested_stack_asset], nested_stack=nested_stack) + stack.sections["Resources"] = dict_section + + collected = self.original_func(stack) + self.assertIn("nested_stack", collected) + self.assertEqual(collected["nested_stack"], nested_stack_asset) + collect_stack_assets_mock.assert_called_once_with(nested_stack) + + +class TestCollectProjectAssets(TestCase): + @patch( + "samcli.lib.iac.cdk.plugin._collect_stack_assets", + side_effect=[{"1": S3Asset(asset_id="1")}, {"2": ImageAsset(asset_id="2")}], + ) + def test_collect_project_assets(self, collect_stack_assets_mock): + project = Project(stacks=[Stack(name="stack1"), Stack(name="stack2")]) + assets, root_stack_names = _collect_project_assets(project) + self.assertIn("stack1", assets) + self.assertIn("1", assets["stack1"]) + self.assertIn("stack2", assets) + self.assertIn("2", assets["stack2"]) + self.assertIn("stack1", root_stack_names) + self.assertIn("stack2", root_stack_names) + + +class TestShallowCloneAsset(TestCase): + def test_shallow_clone_s3_asset(self): + s3_asset = S3Asset() + collected_s3_asset = S3Asset( + source_path="collected_path", + source_property="collected_property", + updated_source_path="collected_updated_path", + destinations=[Mock()], + object_version="collected_version", + object_key="colected_key", + bucket_name="collected_bucket", + ) + collected_assets = {"id": collected_s3_asset} + _shallow_clone_asset(s3_asset, "id", collected_assets) + self.assertNotEqual(s3_asset, collected_s3_asset) + self.assertEqual(s3_asset.source_path, collected_s3_asset.source_path) + self.assertEqual(s3_asset.source_property, collected_s3_asset.source_property) + self.assertEqual(s3_asset.updated_source_path, collected_s3_asset.updated_source_path) + self.assertEqual(s3_asset.destinations, collected_s3_asset.destinations) + self.assertEqual(s3_asset.object_version, collected_s3_asset.object_version) + self.assertEqual(s3_asset.object_key, collected_s3_asset.object_key) + self.assertEqual(s3_asset.bucket_name, collected_s3_asset.bucket_name) + + def test_shallow_clone_image_asset(self): + image_asset = ImageAsset() + collected_image_asset = ImageAsset( + source_local_image="source_local_image", + target="target", + build_args={"foo": "bar"}, + docker_file_name="Dockerfile", + image_tag="tag", + registry="registry", + repository_name="repo", + ) + collected_assets = {"id": collected_image_asset} + _shallow_clone_asset(image_asset, "id", collected_assets) + self.assertNotEqual(image_asset, collected_image_asset) + self.assertEqual(image_asset.source_local_image, collected_image_asset.source_local_image) + self.assertEqual(image_asset.target, collected_image_asset.target) + self.assertEqual(image_asset.build_args, collected_image_asset.build_args) + self.assertEqual(image_asset.docker_file_name, collected_image_asset.docker_file_name) + self.assertEqual(image_asset.image_tag, collected_image_asset.image_tag) + self.assertEqual(image_asset.registry, collected_image_asset.registry) + self.assertEqual(image_asset.repository_name, collected_image_asset.repository_name) + + +class TestUpdateAssetParamsDefaultValues(TestCase): + def test_update_asset_params_default_values(self): + asset = S3Asset( + asset_id="asset", + bucket_name="bucket", + object_key="key", + object_version="version", + extra_details={ + "assetParameters": { + "s3BucketParameter": "xxx", + "s3KeyParameter": "yyy", + "artifactHashParameter": "zzz", + } + }, + ) + + parameters = DictSection("Parameters") + parameters["xxx"] = {"Type": "String"} + parameters["yyy"] = {"Type": "String"} + parameters["zzz"] = {"Type": "String"} + + _update_asset_params_default_values(asset, parameters) + self.assertIn("Default", parameters["xxx"]) + self.assertEqual(parameters["xxx"]["Default"], "bucket") + self.assertIn("Default", parameters["yyy"]) + self.assertEqual(parameters["yyy"]["Default"], "key||version") + self.assertIn("Default", parameters["zzz"]) + self.assertEqual(parameters["zzz"]["Default"], "asset") + + +class TestGetCdkExecutablePath(TestCase): + @patch("platform.system", return_value="windows") + @patch("shutil.which", return_value="cdk.exe") + def test_get_cdk_executable_path_windows(self, which_mock, system_mock): + executable_path = _get_cdk_executable_path() + self.assertEqual(executable_path, "cdk.exe") + + @patch("platform.system", return_value="linux") + @patch("shutil.which", return_value="cdk") + def test_get_cdk_executable_path_non_windows(self, which_mock, system_mock): + executable_path = _get_cdk_executable_path() + self.assertEqual(executable_path, "cdk") + + @patch("platform.system", return_value="linux") + @patch("shutil.which", return_value=None) + def test_get_cdk_executable_path_not_found(self, which_mock, system_mock): + with self.assertRaises(CdkToolkitNotInstalledError): + _get_cdk_executable_path() + + +class TestUpdateBuiltArtifacts(TestCase): + def setUp(self): + self.manifest_dict = { + "artifacts": { + "root-stack-1": { + "metadata": { + "/root-stack-1": [ + { + "type": "aws:cdk:asset", + "data": { + "id": "asset1", + "packaging": "zip", + "path": "original_path", + }, + } + ] + } + }, + "root-stack-2": {}, + } + } + self.updated_manifest = { + "artifacts": { + "root-stack-1": { + "metadata": { + "/root-stack-1": [ + { + "type": "aws:cdk:asset", + "data": { + "id": "asset1", + "packaging": "zip", + "path": "updated_source_path", + }, + } + ] + } + }, + "root-stack-2": {}, + } + } + self.assets = {"root-stack-1": {"asset1": S3Asset(updated_source_path="updated_source_path")}} + self.mock_open = mock_open() + self.collect_project_assets_mock_return_value = (self.assets, ["root-stack-1"]) + + @patch("os.path.join", return_value="path/to/file") + @patch("json.loads") + @patch("json.dumps") + @patch("samcli.lib.iac.cdk.plugin._collect_project_assets") + def test_update_built_artifacts( + self, collect_project_assets_mock, json_dumps_mock, json_loads_mock, os_path_join_mock + ): + json_loads_mock.return_value = self.manifest_dict + with patch("samcli.lib.iac.cdk.plugin.open", self.mock_open): + project = Project(stacks=[]) + collect_project_assets_mock.return_value = self.collect_project_assets_mock_return_value + _update_built_artifacts(project, "cloud_assembly_dir", "build_dir") + json_dumps_mock.assert_called_once_with(self.updated_manifest, indent=4) diff --git a/tests/unit/lib/iac/test_interface.py b/tests/unit/lib/iac/test_interface.py new file mode 100644 index 0000000000..9e3d1dcfb3 --- /dev/null +++ b/tests/unit/lib/iac/test_interface.py @@ -0,0 +1,323 @@ +from copy import deepcopy +from samcli.lib.utils.packagetype import IMAGE, ZIP +from unittest import TestCase +from unittest.mock import patch, Mock, MagicMock + +from samcli.lib.iac.interface import ( + DictSection, + DictSectionItem, + Environment, + Destination, + Asset, + Parameter, + S3Asset, + ImageAsset, + Section, + SectionItem, + SimpleSection, + SimpleSectionItem, + Resource, + Stack, +) + + +class TestEnvironment(TestCase): + def test_properties(self): + env = Environment(region="ap-southeast-1", account_id="012345") + self.assertEqual(env.region, "ap-southeast-1") + self.assertEqual(env.account_id, "012345") + env.region = "us-east-1" + env.account_id = "543210" + self.assertEqual(env.region, "us-east-1") + self.assertEqual(env.account_id, "543210") + + +class TestDestination(TestCase): + def test_properties(self): + destination = Destination(path="path", value="val") + self.assertEqual(destination.path, "path") + self.assertEqual(destination.value, "val") + destination.path = "another_path" + destination.value = "another_val" + self.assertEqual(destination.path, "another_path") + self.assertEqual(destination.value, "another_val") + + +class TestAsset(TestCase): + def test_properties(self): + asset = Asset() + self.assertIsNotNone(asset.asset_id) + self.assertEqual(asset.destinations, []) + self.assertIsNone(asset.source_property) + self.assertEqual(asset.extra_details, {}) + + asset.asset_id = "asset_id" + destionation = Destination(path="path", value="val") + asset.destinations = [destionation] + asset.source_property = "Code" + asset.extra_details = {"foo": "bar"} + self.assertEqual(asset.asset_id, "asset_id") + self.assertEqual(asset.destinations, [destionation]) + self.assertEqual(asset.source_property, "Code") + self.assertEqual(asset.extra_details, {"foo": "bar"}) + + +class TestS3Asset(TestCase): + def test_properties(self): + s3_asset = S3Asset() + self.assertIsNone(s3_asset.bucket_name) + self.assertIsNone(s3_asset.object_key) + self.assertIsNone(s3_asset.object_version) + self.assertIsNone(s3_asset.source_path) + self.assertIsNone(s3_asset.updated_source_path) + + self.assertIsInstance(s3_asset, Asset) + self.assertIsNotNone(s3_asset.asset_id) + self.assertEqual(s3_asset.destinations, []) + self.assertIsNone(s3_asset.source_property) + self.assertEqual(s3_asset.extra_details, {}) + + s3_asset.bucket_name = "bucket" + s3_asset.object_key = "key" + s3_asset.object_version = "v1" + s3_asset.source_path = "path" + s3_asset.updated_source_path = "updated_path" + self.assertEqual(s3_asset.bucket_name, "bucket") + self.assertEqual(s3_asset.object_key, "key") + self.assertEqual(s3_asset.object_version, "v1") + self.assertEqual(s3_asset.source_path, "path") + self.assertEqual(s3_asset.updated_source_path, "updated_path") + + +class TestImageAsset(TestCase): + def test_properties(self): + image_asset = ImageAsset() + self.assertIsNone(image_asset.repository_name) + self.assertIsNone(image_asset.registry) + self.assertIsNone(image_asset.image_tag) + self.assertIsNone(image_asset.source_local_image) + self.assertIsNone(image_asset.source_path) + self.assertIsNone(image_asset.docker_file_name) + self.assertIsNone(image_asset.build_args) + self.assertIsNone(image_asset.target) + + self.assertIsInstance(image_asset, Asset) + self.assertIsNotNone(image_asset.asset_id) + self.assertEqual(image_asset.destinations, []) + self.assertIsNone(image_asset.source_property) + self.assertEqual(image_asset.extra_details, {}) + + image_asset.repository_name = "repo" + image_asset.registry = "registry" + image_asset.image_tag = "tag" + image_asset.source_local_image = "repo:tag" + image_asset.source_path = "path" + image_asset.docker_file_name = "Dockerfile" + image_asset.build_args = {"foo": "bar"} + image_asset.target = "target" + self.assertEqual(image_asset.repository_name, "repo") + self.assertEqual(image_asset.registry, "registry") + self.assertEqual(image_asset.image_tag, "tag") + self.assertEqual(image_asset.source_local_image, "repo:tag") + self.assertEqual(image_asset.source_path, "path") + self.assertEqual(image_asset.docker_file_name, "Dockerfile") + self.assertEqual(image_asset.build_args, {"foo": "bar"}) + self.assertEqual(image_asset.target, "target") + + +class TestSectionItem(TestCase): + def test_properties(self): + section_item = SectionItem() + self.assertIsNone(section_item.key) + self.assertIsNone(section_item.item_id) + + section_item.key = "key" + section_item.item_id = "item_id" + self.assertEqual(section_item.key, "key") + self.assertEqual(section_item.item_id, "item_id") + + +class TestSimpleSectionItem(TestCase): + def test_properties(self): + simple_section_item = SimpleSectionItem() + self.assertIsNone(simple_section_item.value) + self.assertFalse(bool(simple_section_item)) + + self.assertIsInstance(simple_section_item, SectionItem) + self.assertIsNone(simple_section_item.key) + self.assertIsNone(simple_section_item.item_id) + + simple_section_item.value = "some_val" + self.assertEqual(simple_section_item.value, "some_val") + self.assertTrue(bool(simple_section_item)) + + +class TestDictSectionItem(TestCase): + def test_properties(self): + dict_section_item = DictSectionItem() + self.assertEqual(dict_section_item.body, {}) + self.assertEqual(dict_section_item.assets, []) + self.assertEqual(dict_section_item.extra_details, {}) + + self.assertIsInstance(dict_section_item, SectionItem) + self.assertIsNone(dict_section_item.key) + self.assertIsNone(dict_section_item.item_id) + + assets = [Mock()] + dict_section_item.assets = assets + dict_section_item.extra_details = {"foo": "bar"} + self.assertEqual(dict_section_item.assets, assets) + self.assertEqual(dict_section_item.extra_details, {"foo": "bar"}) + + def test_setitem(self): + dict_section_item = DictSectionItem(body={}) + dict_section_item["foo"] = "bar" + self.assertEqual(dict_section_item.body["foo"], "bar") + + def test_delitem(self): + dict_section_item = DictSectionItem(body={"foo": "bar"}) + del dict_section_item["foo"] + self.assertEqual(dict_section_item.body, {}) + + def test_getitem(self): + dict_section_item = DictSectionItem(body={"foo": "bar"}) + self.assertEqual(dict_section_item["foo"], "bar") + + def test_other_mapping_methods(self): + body = {"foo": "bar", "baz": "bax"} + dict_section_item = DictSectionItem(body=body) + self.assertEqual(len(dict_section_item), 2) + for key, val in dict_section_item.items(): + self.assertIn(key, body) + self.assertEqual(val, body[key]) + self.assertTrue(bool(dict_section_item)) + + +class TestSection(TestCase): + def test_properties(self): + section = Section(section_name="name") + self.assertEqual(section.section_name, "name") + + +class TestSimpleSection(TestCase): + def test_properties(self): + simple_section = SimpleSection(section_name="name", value="val") + self.assertEqual(simple_section.section_name, "name") + self.assertEqual(simple_section.value, "val") + simple_section.value = "another_val" + self.assertEqual(simple_section.value, "another_val") + self.assertTrue(bool(simple_section)) + simple_section.value = None + self.assertFalse(bool(simple_section)) + + +class TestDictSection(TestCase): + def test_properties(self): + dict_section_item = DictSectionItem(key="key", body={"foo": "bar"}) + dict_section = DictSection(section_name="name", items=[dict_section_item]) + self.assertEqual(dict_section.section_items, [dict_section_item]) + + @patch("samcli.lib.iac.interface.deepcopy") + def test_copy(self, deepcopy_mock): + expected = Mock() + deepcopy_mock.return_value = expected + dict_section = DictSection(section_name="name", items=[]) + actual = dict_section.copy() + self.assertEqual(actual, expected) + + +class TestResource(TestCase): + def test_properties(self): + stack = Stack() + body = {} + resource = Resource(key="key", body=body) + self.assertIsNone(resource.nested_stack) + resource.nested_stack = stack + self.assertEqual(resource.nested_stack, stack) + + @patch("samcli.lib.iac.interface.deepcopy") + def test_copy(self, deepcopy_mock): + expected = Mock() + deepcopy_mock.return_value = deepcopy_mock.return_value = expected + resource = Resource(key="key") + actual = resource.copy() + self.assertEqual(actual, expected) + + def test_packageable(self): + resource = Resource(key="key") + resource["Type"] = "AWS::Serverless::Function" + self.assertTrue(resource.is_packageable()) + + def test_not_packageable(self): + resource = Resource(key="key") + resource["Type"] = "AWS::APIGateway::Stage" + self.assertFalse(resource.is_packageable()) + + resource = Resource(key="key") + resource["Properties"] = {"InlineCode": "inline_code"} + self.assertFalse(resource.is_packageable()) + + +class TestParamater(TestCase): + def test_properties(self): + parameter = Parameter() + self.assertFalse(parameter.added_by_iac) + parameter.added_by_iac = True + self.assertTrue(parameter.added_by_iac) + + @patch("samcli.lib.iac.interface.deepcopy") + def test_copy(self, deepcopy_mock): + expected = Mock() + deepcopy_mock.return_value = deepcopy_mock.return_value = expected + parameter = Parameter() + actual = parameter.copy() + self.assertEqual(actual, expected) + + +class TestStack(TestCase): + def setUp(self): + asset = S3Asset() + + @patch("samcli.lib.iac.interface.deepcopy") + def test_copy(self, deepcopy_mock): + expected = Mock() + deepcopy_mock.return_value = deepcopy_mock.return_value = expected + stack = Stack() + actual = stack.copy() + self.assertEqual(actual, expected) + + def test_has_assets_of_package_type_zip_true(self): + stack = Stack(assets=[S3Asset()]) + self.assertTrue(stack.has_assets_of_package_type(ZIP)) + + def test_has_assets_of_package_type_zip_false(self): + stack = Stack(assets=[ImageAsset()]) + self.assertFalse(stack.has_assets_of_package_type(ZIP)) + + def test_has_assets_of_package_type_image_true(self): + stack = Stack(assets=[ImageAsset()]) + self.assertTrue(stack.has_assets_of_package_type(IMAGE)) + + def test_has_assets_of_package_type_image_false(self): + stack = Stack(assets=[S3Asset()]) + self.assertFalse(stack.has_assets_of_package_type(IMAGE)) + + def test_find_function_resources_of_package_type_zip(self): + zip_function_resource = Resource(key="zip_function", assets=[S3Asset()]) + zip_function_resource["Type"] = "AWS::Serverless::Function" + image_function_resource = Resource(key="image_function", assets=[ImageAsset()]) + image_function_resource["Type"] = "AWS::Serverless::Function" + resources = DictSection("Resources", items=[zip_function_resource, image_function_resource]) + stack = Stack() + stack["Resources"] = resources + self.assertEqual(stack.find_function_resources_of_package_type(ZIP), [zip_function_resource]) + + def test_find_function_resources_of_package_type_image(self): + zip_function_resource = Resource(key="zip_function", assets=[S3Asset()]) + zip_function_resource["Type"] = "AWS::Serverless::Function" + image_function_resource = Resource(key="image_function", assets=[ImageAsset()]) + image_function_resource["Type"] = "AWS::Serverless::Function" + resources = DictSection("Resources", items=[zip_function_resource, image_function_resource]) + stack = Stack() + stack["Resources"] = resources + self.assertEqual(stack.find_function_resources_of_package_type(IMAGE), [image_function_resource]) diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index f1a793ecf3..e02eab9c2e 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -7,7 +7,7 @@ from contextlib import contextmanager, closing from unittest import mock -from unittest.mock import patch, Mock +from unittest.mock import patch, Mock, MagicMock from samcli.commands.package.exceptions import ExportFailedError from samcli.lib.package.s3_uploader import S3Uploader @@ -18,11 +18,11 @@ from samcli.commands.package import exceptions from samcli.lib.package.artifact_exporter import ( is_local_folder, - make_abs_path, Template, CloudFormationStackResource, ServerlessApplicationResource, ) +from samcli.lib.package.utils import make_abs_path from samcli.lib.package.packageable_resources import ( is_s3_url, is_local_file, @@ -52,6 +52,7 @@ ResourceZip, ResourceImage, ) +from samcli.lib.iac.interface import Stack as IacStack, S3Asset class TestArtifactExporter(unittest.TestCase): @@ -69,6 +70,10 @@ def get_mock(destination: Destination): self.code_signer_mock = Mock() self.code_signer_mock.should_sign_package.return_value = False + self.iac_mock = Mock() + self.iac_mock.should_update_property_after_package = True + self.iac_mock.update_resource_after_packaging = Mock() + def test_all_resources_export(self): uploaded_s3_url = "s3://foo/bar?versionId=baz" @@ -110,11 +115,16 @@ def test_invalid_export_resource(self): s3_uploader_mock = Mock() code_signer_mock = Mock() upload_local_artifacts_mock.reset_mock() - resource_obj = ServerlessFunctionResource(uploaders=self.uploaders_mock, code_signer=code_signer_mock) - resource_id = "id" - resource_dict = {"InlineCode": "code"} + resource_obj = ServerlessFunctionResource( + uploaders=self.uploaders_mock, code_signer=code_signer_mock, iac=self.iac_mock + ) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.get.return_value = {"InlineCode": "code"} + iac_resource_mock.is_packageable.return_value = False + iac_resource_mock.assets = [] parent_dir = "dir" - resource_obj.export(resource_id, resource_dict, parent_dir) + resource_obj.export(iac_resource_mock, parent_dir) upload_local_artifacts_mock.assert_not_called() code_signer_mock.should_sign_package.assert_not_called() code_signer_mock.sign_package.assert_not_called() @@ -131,7 +141,9 @@ def _helper_verify_export_resources( uploaders_mock = Mock() uploaders_mock.get = Mock(return_value=s3_uploader_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] if "." in test_class.PROPERTY_NAME: reversed_property_names = test_class.PROPERTY_NAME.split(".") @@ -139,16 +151,16 @@ def _helper_verify_export_resources( property_dict = {reversed_property_names[0]: "foo"} for sub_property_name in reversed_property_names[1:]: property_dict = {sub_property_name: property_dict} - resource_dict = property_dict + iac_resource_mock["Properties"] = iac_resource_mock.get.return_value = property_dict else: - resource_dict = {test_class.PROPERTY_NAME: "foo"} + iac_resource_mock["Properties"] = iac_resource_mock.get.return_value = {test_class.PROPERTY_NAME: "foo"} parent_dir = "dir" upload_local_artifacts_mock.return_value = uploaded_s3_url - resource_obj = test_class(uploaders=uploaders_mock, code_signer=code_signer_mock) + resource_obj = test_class(uploaders=uploaders_mock, code_signer=code_signer_mock, iac=self.iac_mock) - resource_obj.export(resource_id, resource_dict, parent_dir) + resource_obj.export(iac_resource_mock, parent_dir) if test_class in ( ApiGatewayRestApiResource, @@ -157,18 +169,28 @@ def _helper_verify_export_resources( LambdaLayerVersionResource, ): upload_local_artifacts_mock.assert_called_once_with( - resource_id, resource_dict, test_class.PROPERTY_NAME, parent_dir, s3_uploader_mock + iac_resource_mock.key, + iac_resource_mock.assets[0], + test_class.PROPERTY_NAME, + parent_dir, + s3_uploader_mock, ) else: upload_local_artifacts_mock.assert_called_once_with( - resource_id, resource_dict, test_class.PROPERTY_NAME, parent_dir, s3_uploader_mock, None + iac_resource_mock.key, + iac_resource_mock.assets[0], + test_class.PROPERTY_NAME, + parent_dir, + s3_uploader_mock, + None, ) + self.iac_mock.update_resource_after_packaging.asset_called_once_with(iac_resource_mock) code_signer_mock.sign_package.assert_not_called() if "." in test_class.PROPERTY_NAME: top_level_property_name = test_class.PROPERTY_NAME.split(".")[0] - result = resource_dict[top_level_property_name] + result = iac_resource_mock.get.return_value[top_level_property_name] else: - result = resource_dict[test_class.PROPERTY_NAME] + result = iac_resource_mock.get.return_value[test_class.PROPERTY_NAME] self.assertEqual(result, expected_result) def test_is_s3_url(self): @@ -265,10 +287,9 @@ def test_upload_local_artifacts_local_file(self, zip_and_upload_mock): artifact_path = handle.name parent_dir = tempfile.gettempdir() - resource_dict = {property_name: artifact_path} - result = upload_local_artifacts( - resource_id, resource_dict, property_name, parent_dir, self.s3_uploader_mock - ) + asset_mock = Mock() + asset_mock.source_path = artifact_path + result = upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) self.assertEqual(result, expected_s3_url) # Internally the method would convert relative paths to absolute @@ -291,11 +312,11 @@ def test_upload_local_artifacts_local_file_abs_path(self, zip_and_upload_mock): with tempfile.NamedTemporaryFile() as handle: parent_dir = tempfile.gettempdir() artifact_path = make_abs_path(parent_dir, handle.name) + asset_mock = Mock() + asset_mock.source_path = artifact_path + asset_mock.source_property = property_name - resource_dict = {property_name: artifact_path} - result = upload_local_artifacts( - resource_id, resource_dict, property_name, parent_dir, self.s3_uploader_mock - ) + result = upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) self.assertEqual(result, expected_s3_url) self.s3_uploader_mock.upload_with_dedup.assert_called_with(artifact_path) @@ -313,9 +334,11 @@ def test_upload_local_artifacts_local_folder(self, zip_and_upload_mock): with self.make_temp_dir() as artifact_path: # Artifact is a file in the temporary directory parent_dir = tempfile.gettempdir() - resource_dict = {property_name: artifact_path} + asset_mock = Mock() + asset_mock.source_path = artifact_path + asset_mock.source_property = property_name - result = upload_local_artifacts(resource_id, resource_dict, property_name, parent_dir, Mock()) + result = upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, Mock()) self.assertEqual(result, expected_s3_url) absolute_artifact_path = make_abs_path(parent_dir, artifact_path) @@ -333,8 +356,11 @@ def test_upload_local_artifacts_no_path(self, zip_and_upload_mock): # If you don't specify a path, we will default to Current Working Dir resource_dict = {} parent_dir = tempfile.gettempdir() + asset_mock = Mock() + asset_mock.source_path = None + asset_mock.source_property = property_name - result = upload_local_artifacts(resource_id, resource_dict, property_name, parent_dir, self.s3_uploader_mock) + result = upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) self.assertEqual(result, expected_s3_url) zip_and_upload_mock.assert_called_once_with(parent_dir, mock.ANY, None) @@ -347,10 +373,12 @@ def test_upload_local_artifacts_s3_url(self, zip_and_upload_mock): object_s3_url = "s3://foo/bar?versionId=baz" # If URL is already S3 URL, this will be returned without zip/upload - resource_dict = {property_name: object_s3_url} + asset_mock = Mock() + asset_mock.source_path = object_s3_url + asset_mock.source_property = property_name parent_dir = tempfile.gettempdir() - result = upload_local_artifacts(resource_id, resource_dict, property_name, parent_dir, self.s3_uploader_mock) + result = upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) self.assertEqual(result, object_s3_url) zip_and_upload_mock.assert_not_called() @@ -364,13 +392,17 @@ def test_upload_local_artifacts_invalid_value(self, zip_and_upload_mock): with self.assertRaises(exceptions.InvalidLocalPathError): non_existent_file = "some_random_filename" - resource_dict = {property_name: non_existent_file} - upload_local_artifacts(resource_id, resource_dict, property_name, parent_dir, self.s3_uploader_mock) + asset_mock = Mock() + asset_mock.source_path = non_existent_file + asset_mock.source_property = property_name + upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) with self.assertRaises(exceptions.InvalidLocalPathError): non_existent_file = ["invalid datatype"] - resource_dict = {property_name: non_existent_file} - upload_local_artifacts(resource_id, resource_dict, property_name, parent_dir, self.s3_uploader_mock) + asset_mock = Mock() + asset_mock.source_path = non_existent_file + asset_mock.source_property = property_name + upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) zip_and_upload_mock.assert_not_called() self.s3_uploader_mock.upload_with_dedup.assert_not_called() @@ -393,20 +425,26 @@ def test_resource_zip(self, upload_local_artifacts_mock): class MockResource(ResourceZip): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, self.code_signer_mock) + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = "/path/to/file" + asset_mock.source_property = resource.PROPERTY_NAME resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "/path/to/file" + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://foo/bar" upload_local_artifacts_mock.return_value = s3_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) upload_local_artifacts_mock.assert_called_once_with( - resource_id, resource_dict, resource.PROPERTY_NAME, parent_dir, self.s3_uploader_mock, None + iac_resource_mock.key, asset_mock, resource.PROPERTY_NAME, parent_dir, self.s3_uploader_mock, None ) self.assertEqual(resource_dict[resource.PROPERTY_NAME], s3_url) @@ -418,20 +456,27 @@ def test_resource_lambda_image(self, upload_local_image_artifacts_mock): class MockResource(ResourceImage): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, None) + resource = MockResource(self.uploaders_mock, None, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_local_image = "image:latest" + asset_mock.source_property = resource.PROPERTY_NAME resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "image:latest" + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" ecr_url = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" upload_local_image_artifacts_mock.return_value = ecr_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) upload_local_image_artifacts_mock.assert_called_once_with( - resource_id, resource_dict, resource.PROPERTY_NAME, parent_dir, self.ecr_uploader_mock + iac_resource_mock.key, asset_mock, resource.PROPERTY_NAME, parent_dir, self.ecr_uploader_mock ) self.assertEqual(resource_dict[resource.PROPERTY_NAME], ecr_url) @@ -442,17 +487,24 @@ def test_lambda_image_resource_package_success(self): class MockResource(ResourceImage): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, None) + resource = MockResource(self.uploaders_mock, None, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_local_image = "image:latest" + asset_mock.source_property = resource.PROPERTY_NAME resource_dict = {} original_image = "image:latest" resource_dict[resource.PROPERTY_NAME] = original_image + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" ecr_url = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" self.ecr_uploader_mock.upload.return_value = ecr_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) self.assertEqual(resource_dict[resource.PROPERTY_NAME], ecr_url) @@ -462,15 +514,21 @@ def test_lambda_image_resource_non_package_image_already_remote(self): class MockResource(ResourceImage): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, None) + resource = MockResource(self.uploaders_mock, None, self.iac_mock) - resource_id = "id" - resource_dict = {} original_image = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_local_image = original_image + resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) self.assertEqual(resource_dict[resource.PROPERTY_NAME], original_image) @@ -480,16 +538,22 @@ def test_lambda_image_resource_no_image_present(self): class MockResource(ResourceImage): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, None) + resource = MockResource(self.uploaders_mock, None, self.iac_mock) - resource_id = "id" - resource_dict = {} original_image = None + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_local_image = original_image + resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" with self.assertRaises(ExportFailedError): - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) @patch("shutil.rmtree") @patch("zipfile.is_zipfile") @@ -505,12 +569,18 @@ class MockResource(ResourceZip): PROPERTY_NAME = "foo" FORCE_ZIP = True - resource = MockResource(self.uploaders_mock, self.code_signer_mock) + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" - resource_dict = {} original_path = "/path/to/file" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = original_path + asset_mock.source_property = resource.PROPERTY_NAME + resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_path + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://foo/bar" @@ -524,12 +594,12 @@ class MockResource(ResourceZip): # This is not a zip file is_zipfile_mock.return_value = False - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) zip_and_upload_mock.assert_called_once_with(tmp_dir, mock.ANY, None) rmtree_mock.assert_called_once_with(tmp_dir) is_zipfile_mock.assert_called_once_with(original_path) - self.code_signer_mock.should_sign_package.assert_called_once_with(resource_id) + self.code_signer_mock.should_sign_package.assert_called_once_with(iac_resource_mock.key) self.code_signer_mock.sign_package.assert_not_called() self.assertEqual(resource_dict[resource.PROPERTY_NAME], s3_url) @@ -555,12 +625,18 @@ class MockResource(ResourceZip): PROPERTY_NAME = "foo" FORCE_ZIP = True - resource = MockResource(self.uploaders_mock, self.code_signer_mock) + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" - resource_dict = {} original_path = "/path/to/zip_file" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = original_path + asset_mock.source_property = resource.PROPERTY_NAME + resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_path + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://foo/bar" @@ -571,13 +647,13 @@ class MockResource(ResourceZip): zip_and_upload_mock.return_value = s3_url self.s3_uploader_mock.upload_with_dedup.return_value = s3_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) copy_to_temp_dir_mock.assert_not_called() zip_and_upload_mock.assert_not_called() rmtree_mock.assert_not_called() is_zipfile_mock.assert_called_once_with(original_path) - self.code_signer_mock.should_sign_package.assert_called_once_with(resource_id) + self.code_signer_mock.should_sign_package.assert_called_once_with(iac_resource_mock.key) self.code_signer_mock.sign_package.assert_not_called() self.assertEqual(resource_dict[resource.PROPERTY_NAME], s3_url) @@ -599,12 +675,18 @@ def test_resource_without_force_zip( class MockResourceNoForceZip(ResourceZip): PROPERTY_NAME = "foo" - resource = MockResourceNoForceZip(self.uploaders_mock, self.code_signer_mock) + resource = MockResourceNoForceZip(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" - resource_dict = {} original_path = "/path/to/file" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = original_path + asset_mock.source_property = resource.PROPERTY_NAME + resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_path + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://foo/bar" @@ -615,13 +697,13 @@ class MockResourceNoForceZip(ResourceZip): zip_and_upload_mock.return_value = s3_url self.s3_uploader_mock.upload_with_dedup.return_value = s3_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) copy_to_temp_dir_mock.assert_not_called() zip_and_upload_mock.assert_not_called() rmtree_mock.assert_not_called() is_zipfile_mock.assert_called_once_with(original_path) - self.code_signer_mock.should_sign_package.assert_called_once_with(resource_id) + self.code_signer_mock.should_sign_package.assert_called_once_with(iac_resource_mock.key) self.code_signer_mock.sign_package.assert_not_called() self.assertEqual(resource_dict[resource.PROPERTY_NAME], s3_url) @@ -632,45 +714,28 @@ def test_resource_empty_property_value(self, upload_local_artifacts_mock): class MockResource(ResourceZip): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, self.code_signer_mock) + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = None + asset_mock.source_property = resource.PROPERTY_NAME resource_dict = {} - resource_dict[resource.PROPERTY_NAME] = "/path/to/file" + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://foo/bar" upload_local_artifacts_mock.return_value = s3_url - resource_dict = {} - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) upload_local_artifacts_mock.assert_called_once_with( - resource_id, resource_dict, resource.PROPERTY_NAME, parent_dir, self.s3_uploader_mock, None + iac_resource_mock.key, asset_mock, resource.PROPERTY_NAME, parent_dir, self.s3_uploader_mock, None ) - self.code_signer_mock.should_sign_package.assert_called_once_with(resource_id) + self.code_signer_mock.should_sign_package.assert_called_once_with(iac_resource_mock.key) self.code_signer_mock.sign_package.assert_not_called() self.assertEqual(resource_dict[resource.PROPERTY_NAME], s3_url) - @patch("samcli.lib.package.packageable_resources.upload_local_artifacts") - def test_resource_property_value_dict(self, upload_local_artifacts_mock): - # Property value is a dictionary. Export should not upload anything - - class MockResource(ResourceZip): - PROPERTY_NAME = "foo" - - resource = MockResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" - resource_dict = {} - resource_dict[resource.PROPERTY_NAME] = "/path/to/file" - parent_dir = "dir" - s3_url = "s3://foo/bar" - - upload_local_artifacts_mock.return_value = s3_url - resource_dict = {} - resource_dict[resource.PROPERTY_NAME] = {"a": "b"} - resource.export(resource_id, resource_dict, parent_dir) - upload_local_artifacts_mock.assert_not_called() - self.assertEqual(resource_dict, {"foo": {"a": "b"}}) - @patch("samcli.lib.package.packageable_resources.upload_local_artifacts") def test_resource_has_package_null_property_to_false(self, upload_local_artifacts_mock): # Should not upload anything if PACKAGE_NULL_PROPERTY is set to False @@ -679,15 +744,21 @@ class MockResource(ResourceZip): PROPERTY_NAME = "foo" PACKAGE_NULL_PROPERTY = False - resource = MockResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = None resource_dict = {} + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://foo/bar" upload_local_artifacts_mock.return_value = s3_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) upload_local_artifacts_mock.assert_not_called() self.assertNotIn(resource.PROPERTY_NAME, resource_dict) @@ -697,18 +768,19 @@ def test_resource_export_fails(self, upload_local_artifacts_mock): class MockResource(ResourceZip): PROPERTY_NAME = "foo" - resource = MockResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" - resource_dict = {} - resource_dict[resource.PROPERTY_NAME] = "/path/to/file" + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + iac_resource_mock.key = "id" parent_dir = "dir" s3_url = "s3://foo/bar" upload_local_artifacts_mock.side_effect = RuntimeError resource_dict = {} + iac_resource_mock.get.return_value = resource_dict with self.assertRaises(exceptions.ExportFailedError): - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) @patch("samcli.lib.package.packageable_resources.upload_local_artifacts") def test_resource_with_s3_url_dict(self, upload_local_artifacts_mock): @@ -725,21 +797,28 @@ class MockResource(ResourceWithS3UrlDict): OBJECT_KEY_PROPERTY = "o" VERSION_PROPERTY = "v" - resource = MockResource(self.uploaders_mock, self.code_signer_mock) + resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) # Case 1: Property value is a path to file - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = "/path/to/file" + asset_mock.source_property = resource.PROPERTY_NAME resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "/path/to/file" + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" s3_url = "s3://bucket/key1/key2?versionId=SomeVersionNumber" upload_local_artifacts_mock.return_value = s3_url - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) upload_local_artifacts_mock.assert_called_once_with( - resource_id, resource_dict, resource.PROPERTY_NAME, parent_dir, self.s3_uploader_mock + iac_resource_mock.key, asset_mock, resource.PROPERTY_NAME, parent_dir, self.s3_uploader_mock ) self.assertEqual( @@ -756,19 +835,26 @@ class MockResource(ResourceZip): code_signer_mock.sign_package.return_value = "signed_s3_location" upload_local_artifacts_mock.return_value = "non_signed_s3_location" - resource = MockResource(self.uploaders_mock, code_signer_mock) + resource = MockResource(self.uploaders_mock, code_signer_mock, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = "/path/to/file" + asset_mock.source_property = resource.PROPERTY_NAME resource_dict = {resource.PROPERTY_NAME: "/path/to/file"} + iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" - resource.export(resource_id, resource_dict, parent_dir) + resource.export(iac_resource_mock, parent_dir) self.assertEqual(resource_dict[resource.PROPERTY_NAME], "signed_s3_location") @patch("samcli.lib.package.artifact_exporter.Template") def test_export_cloudformation_stack(self, TemplateMock): - stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock) + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME exported_template_dict = {"foo": "bar"} result_s3_url = "s3://hello/world" @@ -784,85 +870,135 @@ def test_export_cloudformation_stack(self, TemplateMock): with tempfile.NamedTemporaryFile() as handle: template_path = handle.name resource_dict = {property_name: template_path} + iac_resource_mock.get.return_value = resource_dict parent_dir = tempfile.gettempdir() - stack_resource.export(resource_id, resource_dict, parent_dir) + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = template_path + asset_mock.source_property = property_name + stack_resource.export(iac_resource_mock, parent_dir) self.assertEqual(resource_dict[property_name], result_path_style_s3_url) - TemplateMock.assert_called_once_with(template_path, parent_dir, self.uploaders_mock, self.code_signer_mock) + TemplateMock.assert_called_once_with( + nested_stack_mock, parent_dir, self.uploaders_mock, self.code_signer_mock, self.iac_mock + ) template_instance_mock.export.assert_called_once_with() self.s3_uploader_mock.upload_with_dedup.assert_called_once_with(mock.ANY, "template") self.s3_uploader_mock.to_path_style_s3_url.assert_called_once_with("world", None) + def test_export_cloudformation_stack_no_nested_stack(self): + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + do_export_mock = Mock() + stack_resource.do_export = do_export_mock + + iac_resource_mock = MagicMock() + iac_resource_mock.nested_stack = None + + stack_resource.export(iac_resource_mock, "dir") + do_export_mock.assert_not_called() + def test_export_cloudformation_stack_no_upload_path_is_s3url(self): - stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" resource_dict = {property_name: s3_url} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock # Case 1: Path is already S3 url - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict[property_name], s3_url) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_cloudformation_stack_no_upload_path_is_httpsurl(self): - stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "https://s3.amazonaws.com/hello/world" resource_dict = {property_name: s3_url} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock # Case 1: Path is already S3 url - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict[property_name], s3_url) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_cloudformation_stack_no_upload_path_is_s3_region_httpsurl(self): - stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" - property_name = stack_resource.PROPERTY_NAME + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + property_name = stack_resource.PROPERTY_NAME s3_url = "https://s3.some-valid-region.amazonaws.com/hello/world" resource_dict = {property_name: s3_url} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict[property_name], s3_url) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_cloudformation_stack_no_upload_path_is_empty(self): - stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" resource_dict = {property_name: s3_url} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock # Case 2: Path is empty resource_dict = {} - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict, {}) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_cloudformation_stack_no_upload_path_not_file(self): - stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock # Case 3: Path is not a file with self.make_temp_dir() as dirname: + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] + asset_mock.source_path = dirname + asset_mock.source_property = property_name resource_dict = {property_name: dirname} + iac_resource_mock.get.return_value = resource_dict with self.assertRaises(exceptions.ExportFailedError): - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.s3_uploader_mock.upload_with_dedup.assert_not_called() @patch("samcli.lib.package.artifact_exporter.Template") def test_export_serverless_application(self, TemplateMock): - stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock) + stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - resource_id = "id" + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] exported_template_dict = {"foo": "bar"} result_s3_url = "s3://hello/world" result_path_style_s3_url = "http://s3.amazonws.com/hello/world" @@ -876,83 +1012,112 @@ def test_export_serverless_application(self, TemplateMock): with tempfile.NamedTemporaryFile() as handle: template_path = handle.name + asset_mock.source_path = template_path + asset_mock.source_property = property_name resource_dict = {property_name: template_path} + iac_resource_mock.get.return_value = resource_dict parent_dir = tempfile.gettempdir() + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock - stack_resource.export(resource_id, resource_dict, parent_dir) + stack_resource.export(iac_resource_mock, parent_dir) self.assertEqual(resource_dict[property_name], result_path_style_s3_url) - TemplateMock.assert_called_once_with(template_path, parent_dir, self.uploaders_mock, self.code_signer_mock) + TemplateMock.assert_called_once_with( + nested_stack_mock, parent_dir, self.uploaders_mock, self.code_signer_mock, self.iac_mock + ) template_instance_mock.export.assert_called_once_with() self.s3_uploader_mock.upload_with_dedup.assert_called_once_with(mock.ANY, "template") self.s3_uploader_mock.to_path_style_s3_url.assert_called_once_with("world", None) def test_export_serverless_application_no_upload_path_is_s3url(self): - stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" resource_dict = {property_name: s3_url} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock # Case 1: Path is already S3 url - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict[property_name], s3_url) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_serverless_application_no_upload_path_is_httpsurl(self): - stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "https://s3.amazonaws.com/hello/world" resource_dict = {property_name: s3_url} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock # Case 1: Path is already S3 url - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict[property_name], s3_url) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_serverless_application_no_upload_path_is_empty(self): - stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME # Case 2: Path is empty resource_dict = {} - stack_resource.export(resource_id, resource_dict, "dir") + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict, {}) self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_serverless_application_no_upload_path_not_file(self): - stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" + iac_resource_mock.assets = MagicMock() + iac_resource_mock.assets[0] = Mock() + asset_mock = iac_resource_mock.assets[0] property_name = stack_resource.PROPERTY_NAME # Case 3: Path is not a file with self.make_temp_dir() as dirname: + asset_mock.source_path = dirname + asset_mock.source_property = property_name resource_dict = {property_name: dirname} + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock with self.assertRaises(exceptions.ExportFailedError): - stack_resource.export(resource_id, resource_dict, "dir") + stack_resource.export(iac_resource_mock, "dir") self.s3_uploader_mock.upload_with_dedup.assert_not_called() def test_export_serverless_application_no_upload_path_is_dictionary(self): - stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock) - resource_id = "id" + stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + iac_resource_mock = MagicMock() + iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME # Case 4: Path is dictionary location = {"ApplicationId": "id", "SemanticVersion": "1.0.1"} resource_dict = {property_name: location} - stack_resource.export(resource_id, resource_dict, "dir") + iac_resource_mock.get.return_value = resource_dict + nested_stack_mock = Mock() + iac_resource_mock.nested_stack = nested_stack_mock + stack_resource.export(iac_resource_mock, "dir") self.assertEqual(resource_dict[property_name], location) self.s3_uploader_mock.upload_with_dedup.assert_not_called() - @patch("samcli.lib.package.artifact_exporter.yaml_parse") - def test_template_export_metadata(self, yaml_parse_mock): + def test_template_export_metadata(self): parent_dir = os.path.sep template_dir = os.path.join(parent_dir, "foo", "bar") - template_path = os.path.join(template_dir, "path") - template_str = self.example_yaml_template() metadata_type1_class = Mock() metadata_type1_class.RESOURCE_TYPE = "metadata_type1" @@ -974,37 +1139,29 @@ def test_template_export_metadata(self, yaml_parse_mock): metadata_to_export = [metadata_type1_class, metadata_type2_class] template_dict = {"Metadata": {"metadata_type1": {"property_1": "abc"}, "metadata_type2": {"property_2": "def"}}} - open_mock = mock.mock_open() - yaml_parse_mock.return_value = template_dict - - # Patch the file open method to return template string - with patch("samcli.lib.package.artifact_exporter.open", open_mock(read_data=template_str)) as open_mock: - - template_exporter = Template( - template_path, - parent_dir, - self.uploaders_mock, - self.code_signer_mock, - metadata_to_export=metadata_to_export, - ) - exported_template = template_exporter.export() - self.assertEqual(exported_template, template_dict) - - open_mock.assert_called_once_with(make_abs_path(parent_dir, template_path), "r") - - self.assertEqual(1, yaml_parse_mock.call_count) + nested_stack_mock = IacStack() + nested_stack_mock.update(template_dict) + nested_stack_mock.origin_dir = template_dir + + template_exporter = Template( + nested_stack_mock, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + self.iac_mock, + metadata_to_export=metadata_to_export, + ) + exported_template = template_exporter.export() + self.assertEqual(exported_template, template_dict) - metadata_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) - metadata_type1_instance.export.assert_called_once_with("metadata_type1", mock.ANY, template_dir) - metadata_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) - metadata_type2_instance.export.assert_called_once_with("metadata_type2", mock.ANY, template_dir) + metadata_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) + metadata_type1_instance.export.assert_called_once_with("metadata_type1", mock.ANY, template_dir) + metadata_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) + metadata_type2_instance.export.assert_called_once_with("metadata_type2", mock.ANY, template_dir) - @patch("samcli.lib.package.artifact_exporter.yaml_parse") - def test_template_export(self, yaml_parse_mock): + def test_template_export(self): parent_dir = os.path.sep template_dir = os.path.join(parent_dir, "foo", "bar") - template_path = os.path.join(template_dir, "path") - template_str = self.example_yaml_template() resource_type1_class = Mock() resource_type1_class.RESOURCE_TYPE = "resource_type1" @@ -1029,34 +1186,32 @@ def test_template_export(self, yaml_parse_mock): "Resource3": {"Type": "some-other-type", "Properties": properties}, } } + nested_stack_mock = IacStack() + nested_stack_mock.update(template_dict) + nested_stack_mock.origin_dir = template_dir + + template_exporter = Template( + nested_stack_mock, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + self.iac_mock, + resources_to_export, + ) + exported_template = template_exporter.export() + self.assertEqual(exported_template, template_dict) - open_mock = mock.mock_open() - yaml_parse_mock.return_value = template_dict - - # Patch the file open method to return template string - with patch("samcli.lib.package.artifact_exporter.open", open_mock(read_data=template_str)) as open_mock: - - template_exporter = Template( - template_path, parent_dir, self.uploaders_mock, self.code_signer_mock, resources_to_export - ) - exported_template = template_exporter.export() - self.assertEqual(exported_template, template_dict) - - open_mock.assert_called_once_with(make_abs_path(parent_dir, template_path), "r") - - self.assertEqual(1, yaml_parse_mock.call_count) + iac_resource1 = nested_stack_mock["Resources"]["Resource1"] + iac_resource2 = nested_stack_mock["Resources"]["Resource2"] - resource_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) - resource_type1_instance.export.assert_called_once_with("Resource1", mock.ANY, template_dir) - resource_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) - resource_type2_instance.export.assert_called_once_with("Resource2", mock.ANY, template_dir) + resource_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + resource_type1_instance.export.assert_called_once_with(iac_resource1, template_dir) + resource_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, self.iac_mock) + resource_type2_instance.export.assert_called_once_with(iac_resource2, template_dir) - @patch("samcli.lib.package.artifact_exporter.yaml_parse") - def test_template_export_with_globals(self, yaml_parse_mock): + def test_template_export_with_globals(self): parent_dir = os.path.sep template_dir = os.path.join(parent_dir, "foo", "bar") - template_path = os.path.join(template_dir, "path") - template_str = self.example_yaml_template() resource_type1_class = Mock() resource_type1_class.RESOURCE_TYPE = "resource_type1" @@ -1083,28 +1238,27 @@ def test_template_export_with_globals(self, yaml_parse_mock): } }, } + nested_stack_mock = IacStack() + nested_stack_mock.update(template_dict) + nested_stack_mock.origin_dir = template_dir + + template_exporter = Template( + nested_stack_mock, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + self.iac_mock, + resources_to_export, + ) + exported_template = template_exporter.export() + self.assertEqual(exported_template, template_dict) + self.assertEqual( + exported_template["Resources"]["FunResource"]["Properties"]["CodeUri"], "s3://test-bucket/test-key" + ) - open_mock = mock.mock_open() - yaml_parse_mock.return_value = template_dict - - # Patch the file open method to return template string - with patch("samcli.lib.package.artifact_exporter.open", open_mock(read_data=template_str)) as open_mock: - - template_exporter = Template( - template_path, parent_dir, self.uploaders_mock, self.code_signer_mock, resources_to_export - ) - exported_template = template_exporter.export() - self.assertEqual(exported_template, template_dict) - self.assertEqual( - exported_template["Resources"]["FunResource"]["Properties"]["CodeUri"], "s3://test-bucket/test-key" - ) - - @patch("samcli.lib.package.artifact_exporter.yaml_parse") - def test_template_global_export(self, yaml_parse_mock): + def test_template_global_export(self): parent_dir = os.path.sep template_dir = os.path.join(parent_dir, "foo", "bar") - template_path = os.path.join(template_dir, "path") - template_str = self.example_yaml_template() resource_type1_class = Mock() resource_type1_class.RESOURCE_TYPE = "resource_type1" @@ -1130,40 +1284,48 @@ def test_template_global_export(self, yaml_parse_mock): }, "List": ["foo", properties_in_list], } - open_mock = mock.mock_open() + nested_stack_mock = IacStack() + nested_stack_mock.update(template_dict) + nested_stack_mock.origin_dir = template_dir + include_transform_export_handler_mock = Mock() include_transform_export_handler_mock.return_value = { "Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}, } - yaml_parse_mock.return_value = template_dict - - with patch("samcli.lib.package.artifact_exporter.open", open_mock(read_data=template_str)) as open_mock: - with patch.dict(GLOBAL_EXPORT_DICT, {"Fn::Transform": include_transform_export_handler_mock}): - - template_exporter = Template(template_path, parent_dir, self.uploaders_mock, resources_to_export) - exported_template = template_exporter._export_global_artifacts(template_exporter.template_dict) - - first_call_args, kwargs = include_transform_export_handler_mock.call_args_list[0] - second_call_args, kwargs = include_transform_export_handler_mock.call_args_list[1] - third_call_args, kwargs = include_transform_export_handler_mock.call_args_list[2] - call_args = [first_call_args[0], second_call_args[0], third_call_args[0]] - self.assertTrue({"Name": "AWS::Include", "Parameters": {"Location": "foo.yaml"}} in call_args) - self.assertTrue({"Name": "AWS::OtherTransform"} in call_args) - self.assertTrue({"Name": "AWS::Include", "Parameters": {"Location": "bar.yaml"}} in call_args) - self.assertTrue(template_dir in first_call_args) - self.assertTrue(template_dir in second_call_args) - self.assertTrue(template_dir in third_call_args) - self.assertEqual(include_transform_export_handler_mock.call_count, 3) - # new s3 url is added to include location - self.assertEqual( - exported_template["Resources"]["Resource1"]["Properties"]["Fn::Transform"], - {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}}, - ) - self.assertEqual( - exported_template["List"][1]["Fn::Transform"], - {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}}, - ) + + with patch.dict(GLOBAL_EXPORT_DICT, {"Fn::Transform": include_transform_export_handler_mock}): + + template_exporter = Template( + nested_stack_mock, + parent_dir, + self.uploaders_mock, + self.code_signer_mock, + self.iac_mock, + resources_to_export, + ) + template_exporter._export_global_artifacts(template_exporter.template_dict) + + first_call_args, kwargs = include_transform_export_handler_mock.call_args_list[0] + second_call_args, kwargs = include_transform_export_handler_mock.call_args_list[1] + third_call_args, kwargs = include_transform_export_handler_mock.call_args_list[2] + call_args = [first_call_args[0], second_call_args[0], third_call_args[0]] + self.assertTrue({"Name": "AWS::Include", "Parameters": {"Location": "foo.yaml"}} in call_args) + self.assertTrue({"Name": "AWS::OtherTransform"} in call_args) + self.assertTrue({"Name": "AWS::Include", "Parameters": {"Location": "bar.yaml"}} in call_args) + self.assertTrue(template_dir in first_call_args) + self.assertTrue(template_dir in second_call_args) + self.assertTrue(template_dir in third_call_args) + self.assertEqual(include_transform_export_handler_mock.call_count, 3) + # new s3 url is added to include location + self.assertEqual( + template_exporter.template_dict["Resources"]["Resource1"]["Properties"]["Fn::Transform"], + {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}}, + ) + self.assertEqual( + template_exporter.template_dict["List"][1]["Fn::Transform"], + {"Name": "AWS::Include", "Parameters": {"Location": "s3://foo"}}, + ) @patch("samcli.lib.package.packageable_resources.is_local_file") def test_include_transform_export_handler_with_relative_file_path(self, is_local_file_mock): @@ -1262,16 +1424,18 @@ def test_include_transform_export_handler_non_include_transform(self, is_local_f def test_template_export_path_be_folder(self): - template_path = "/path/foo" + iac_stack_mock = Mock() # Set parent_dir to be a non-existent folder with self.assertRaises(ValueError): - Template(template_path, "somefolder", self.uploaders_mock, self.code_signer_mock) + Template(iac_stack_mock, "somefolder", self.uploaders_mock, self.code_signer_mock, self.iac_mock) # Set parent_dir to be a real folder, but just a relative path with self.make_temp_dir() as dirname: with self.assertRaises(ValueError): - Template(template_path, os.path.relpath(dirname), self.uploaders_mock, self.code_signer_mock) + Template( + iac_stack_mock, os.path.relpath(dirname), self.uploaders_mock, self.code_signer_mock, self.iac_mock + ) def test_make_zip(self): test_file_creator = FileCreator() From 723b6416587c4562323b22ac426dfab40a0e338f Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 29 Jun 2021 13:21:56 -0700 Subject: [PATCH 02/38] SAM CLI CDK init flow: (#344) * SAM CLI CDK init flow: interactive and no interactive * fix current test cases * black reformat * Allow clone from non-master branch * trigger tests * Resolve comments * Resolve comments, fix cdk runtime list, and improve docstring and error message * fix pylint * fix pylint * Update exception name for CDK project errors --- samcli/commands/init/__init__.py | 44 ++++- samcli/commands/init/init_generator.py | 24 ++- samcli/commands/init/init_templates.py | 69 ++++--- samcli/commands/init/interactive_init_flow.py | 108 ++++++++--- samcli/lib/iac/cdk/cloud_assembly.py | 4 +- samcli/lib/init/__init__.py | 13 +- samcli/local/common/runtime_template.py | 26 +++ tests/integration/buildcmd/test_build_cmd.py | 12 +- tests/unit/commands/_utils/test_options.py | 6 +- tests/unit/commands/init/test_cli.py | 170 +++++++++++++++++- tests/unit/commands/init/test_templates.py | 12 +- .../unit/commands/samconfig/test_samconfig.py | 3 + 12 files changed, 422 insertions(+), 69 deletions(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index ebf1d18fff..626a9996bc 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -10,8 +10,14 @@ from samcli.cli.cli_config_file import configuration_option, TomlProvider from samcli.cli.main import pass_context, common_options, print_cmdline_args +from samcli.lib.iac.interface import ProjectTypes from samcli.lib.utils.version_checker import check_newer_version -from samcli.local.common.runtime_template import RUNTIMES, SUPPORTED_DEP_MANAGERS, LAMBDA_IMAGES_RUNTIMES +from samcli.local.common.runtime_template import ( + RUNTIMES, + SUPPORTED_DEP_MANAGERS, + LAMBDA_IMAGES_RUNTIMES, + INIT_CDK_LANGUAGES, +) from samcli.lib.telemetry.metric import track_command from samcli.commands.init.interactive_init_flow import _get_runtime_from_image from samcli.commands.local.cli_common.click_mutex import Mutex @@ -147,6 +153,16 @@ def wrapped(*args, **kwargs): cls=Mutex, not_required=["location", "base_image"], ) +@click.option( + "--project-type", + help="Project Type of your app, CDK or CFN", + type=click.Choice(ProjectTypes.__members__, case_sensitive=False), +) +@click.option( + "--cdk-language", + help="CDK project language of your app, for CDK project only", + type=click.Choice(INIT_CDK_LANGUAGES), +) @click.option( "-p", "--package-type", @@ -220,6 +236,8 @@ def cli( extra_context, config_file, config_env, + project_type, + cdk_language, ): """ `sam init` command entry point @@ -238,6 +256,8 @@ def cli( app_template, no_input, extra_context, + project_type, + cdk_language, ) # pragma: no cover @@ -256,6 +276,8 @@ def do_cli( app_template, no_input, extra_context, + project_type, + cdk_language, auto_clone=True, ): """ @@ -277,7 +299,9 @@ def do_cli( templates = InitTemplates(no_interactive, auto_clone) if package_type == IMAGE and image_bool: base_image, runtime = _get_runtime_from_image(base_image) - options = templates.init_options(package_type, runtime, base_image, dependency_manager) + options = templates.init_options( + project_type, cdk_language, package_type, runtime, base_image, dependency_manager + ) if len(options) == 1: app_template = options[0].get("appTemplate") elif len(options) > 1: @@ -288,17 +312,29 @@ def do_cli( if app_template and not location: location = templates.location_from_app_template( - package_type, runtime, base_image, dependency_manager, app_template + project_type, cdk_language, package_type, runtime, base_image, dependency_manager, app_template ) no_input = True extra_context = _get_cookiecutter_template_context(name, runtime, extra_context) if not output_dir: output_dir = "." - do_generate(location, package_type, runtime, dependency_manager, output_dir, name, no_input, extra_context) + do_generate( + project_type, + location, + package_type, + runtime, + dependency_manager, + output_dir, + name, + no_input, + extra_context, + ) else: # proceed to interactive state machine, which will call do_generate do_interactive( + project_type, + cdk_language, location, pt_explicit, package_type, diff --git a/samcli/commands/init/init_generator.py b/samcli/commands/init/init_generator.py index c8de6d10a9..892d26833b 100644 --- a/samcli/commands/init/init_generator.py +++ b/samcli/commands/init/init_generator.py @@ -7,8 +7,28 @@ from samcli.lib.init.exceptions import InitErrorException -def do_generate(location, package_type, runtime, dependency_manager, output_dir, name, no_input, extra_context): +def do_generate( + project_type, + location, + package_type, + runtime, + dependency_manager, + output_dir, + name, + no_input, + extra_context, +): try: - generate_project(location, package_type, runtime, dependency_manager, output_dir, name, no_input, extra_context) + generate_project( + project_type, + location, + package_type, + runtime, + dependency_manager, + output_dir, + name, + no_input, + extra_context, + ) except InitErrorException as e: raise UserException(str(e), wrapped_from=e.__class__.__name__) from e diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index 303a2632af..f1ac5bf64e 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -17,6 +17,7 @@ from samcli.cli.main import global_cfg from samcli.commands.exceptions import UserException, AppTemplateUpdateException +from samcli.lib.iac.interface import ProjectTypes from samcli.lib.utils import osutils from samcli.lib.utils.osutils import rmtree_callback from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING, get_local_lambda_images_location @@ -28,10 +29,14 @@ class InvalidInitTemplateError(UserException): pass +class CDKProjectInvalidInitConfigError(UserException): + pass + class InitTemplates: def __init__(self, no_interactive=False, auto_clone=True): self._repo_url = "https://github.com/aws/aws-sam-cli-app-templates" + self._repo_branch = "cdk-template" self._repo_name = "aws-sam-cli-app-templates" self._temp_repo_name = "TEMP-aws-sam-cli-app-templates" self.repo_path = None @@ -39,12 +44,16 @@ def __init__(self, no_interactive=False, auto_clone=True): self._no_interactive = no_interactive self._auto_clone = auto_clone - def prompt_for_location(self, package_type, runtime, base_image, dependency_manager): + def prompt_for_location(self, project_type, cdk_language, package_type, runtime, base_image, dependency_manager): """ Prompt for template location based on other information provided in previous steps. Parameters ---------- + project_type : ProjectTypes + the type of project, CFN or CDK + cdk_language : Optional[str] + the language of cdk app package_type : str the package type, 'Zip' or 'Image', see samcli/lib/utils/packagetype.py runtime : str @@ -61,7 +70,7 @@ def prompt_for_location(self, package_type, runtime, base_image, dependency_mana app_template : str The name of the template """ - options = self.init_options(package_type, runtime, base_image, dependency_manager) + options = self.init_options(project_type, cdk_language, package_type, runtime, base_image, dependency_manager) if len(options) == 1: template_md = options[0] @@ -92,8 +101,10 @@ def prompt_for_location(self, package_type, runtime, base_image, dependency_mana return (os.path.join(self.repo_path, template_md["directory"]), template_md["appTemplate"]) raise InvalidInitTemplateError("Invalid template. This should not be possible, please raise an issue.") - def location_from_app_template(self, package_type, runtime, base_image, dependency_manager, app_template): - options = self.init_options(package_type, runtime, base_image, dependency_manager) + def location_from_app_template( + self, project_type, cdk_language, package_type, runtime, base_image, dependency_manager, app_template + ): + options = self.init_options(project_type, cdk_language, package_type, runtime, base_image, dependency_manager) try: template = next(item for item in options if self._check_app_template(item, app_template)) if template.get("init_location") is not None: @@ -111,27 +122,39 @@ def _check_app_template(entry: Dict, app_template: str) -> bool: # detail: https://github.com/python/mypy/issues/5697 return bool(entry["appTemplate"] == app_template) - def init_options(self, package_type, runtime, base_image, dependency_manager): + def init_options(self, project_type, cdk_language, package_type, runtime, base_image, dependency_manager): if not self.clone_attempted: self._clone_repo() if self.repo_path is None: - return self._init_options_from_bundle(package_type, runtime, dependency_manager) - return self._init_options_from_manifest(package_type, runtime, base_image, dependency_manager) + return self._init_options_from_bundle(project_type, cdk_language, package_type, runtime, dependency_manager) + return self._init_options_from_manifest( + project_type, cdk_language, package_type, runtime, base_image, dependency_manager + ) - def _init_options_from_manifest(self, package_type, runtime, base_image, dependency_manager): + def _init_options_from_manifest( + self, project_type, cdk_language, package_type, runtime, base_image, dependency_manager + ): manifest_path = os.path.join(self.repo_path, "manifest.json") with open(str(manifest_path)) as fp: body = fp.read() manifest_body = json.loads(body) templates = None - if base_image: + if project_type == ProjectTypes.CDK: + if cdk_language and runtime: + templates = manifest_body.get(f"cdk-{cdk_language}", {}).get(runtime) + else: + msg = f"CDK language and runtime are necessary in project type: {project_type.value}." + raise CDKProjectInvalidInitConfigError(msg) + elif base_image: templates = manifest_body.get(base_image) elif runtime: templates = manifest_body.get(runtime) if templates is None: # Fallback to bundled templates - return self._init_options_from_bundle(package_type, runtime, dependency_manager) + return self._init_options_from_bundle( + project_type, cdk_language, package_type, runtime, dependency_manager + ) if dependency_manager is not None: templates_by_dep = filter(lambda x: x["dependencyManager"] == dependency_manager, list(templates)) @@ -139,8 +162,9 @@ def _init_options_from_manifest(self, package_type, runtime, base_image, depende return list(templates) @staticmethod - def _init_options_from_bundle(package_type, runtime, dependency_manager): - for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): + def _init_options_from_bundle(project_type, cdk_language, package_type, runtime, dependency_manager): + runtime_dependency_template_mapping = {} if project_type == ProjectTypes.CDK else RUNTIME_DEP_TEMPLATE_MAPPING + for mapping in list(itertools.chain(*(runtime_dependency_template_mapping.values()))): if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): if not dependency_manager or dependency_manager == mapping["dependency_manager"]: if package_type == IMAGE: @@ -187,7 +211,8 @@ def _overwrite_existing_templates(self, expected_path: str): expected_temp_path = os.path.normpath(os.path.join(tempdir, self._repo_name)) LOG.info("\nCloning app templates from %s", self._repo_url) subprocess.check_output( - [self._git_executable(), "clone", self._repo_url, self._repo_name], + # TODO: [UPDATEME] wchengru: We should remove --branch option when making CDK support GA. + [self._git_executable(), "clone", "--branch", self._repo_branch, self._repo_url, self._repo_name], cwd=tempdir, stderr=subprocess.STDOUT, ) @@ -257,18 +282,22 @@ def _git_executable() -> str: LOG.debug("Unable to find executable %s", name, exc_info=ex) raise OSError("Cannot find git, was looking at executables: {}".format(options)) - def is_dynamic_schemas_template(self, package_type, app_template, runtime, base_image, dependency_manager): + def is_dynamic_schemas_template( + self, project_type, cdk_language, package_type, app_template, runtime, base_image, dependency_manager + ): """ Check if provided template is dynamic template e.g: AWS Schemas template. Currently dynamic templates require different handling e.g: for schema download & merge schema code in sam-app. - :param package_type: - :param app_template: - :param runtime: - :param base_image: - :param dependency_manager: + :param project_type: SAM supported project type, ProjectTypes.CDK or ProjectTypes.CFN + :param cdk_language: CDK stacks definition language. + :param package_type: Template package type: ZIP or IMAGE. + :param app_template: Identifier of the managed application template + :param runtime: Lambda funtion runtime + :param base_image: Runtime base image + :param dependency_manager: Runtime dependency manager. :return: """ - options = self.init_options(package_type, runtime, base_image, dependency_manager) + options = self.init_options(project_type, cdk_language, package_type, runtime, base_image, dependency_manager) for option in options: if option.get("appTemplate") == app_template: return option.get("isDynamicTemplate", False) diff --git a/samcli/commands/init/interactive_init_flow.py b/samcli/commands/init/interactive_init_flow.py index ae0bd2d3c9..44194a2d09 100644 --- a/samcli/commands/init/interactive_init_flow.py +++ b/samcli/commands/init/interactive_init_flow.py @@ -13,8 +13,15 @@ get_schemas_template_parameter, ) from samcli.commands.exceptions import SchemasApiException +from samcli.lib.iac.interface import ProjectTypes from samcli.lib.schemas.schemas_code_manager import do_download_source_code_binding, do_extract_and_merge_schemas_code -from samcli.local.common.runtime_template import INIT_RUNTIMES, RUNTIME_TO_DEPENDENCY_MANAGERS, LAMBDA_IMAGES_RUNTIMES +from samcli.local.common.runtime_template import ( + INIT_RUNTIMES, + RUNTIME_TO_DEPENDENCY_MANAGERS, + LAMBDA_IMAGES_RUNTIMES, + INIT_CDK_LANGUAGES, + CDK_INIT_RUNTIMES, +) from samcli.commands.init.init_generator import do_generate from samcli.commands.init.init_templates import InitTemplates from samcli.lib.utils.osutils import remove @@ -24,6 +31,8 @@ def do_interactive( + project_type, + cdk_language, location, pt_explicit, package_type, @@ -39,6 +48,7 @@ def do_interactive( Implementation of the ``cli`` method when --interactive is provided. It will ask customers a few questions to init a template. """ + project_type = _get_project_type(project_type) if app_template: location_opt_choice = "1" else: @@ -47,26 +57,40 @@ def do_interactive( location_opt_choice = click.prompt("Choice", type=click.Choice(["1", "2"]), show_choices=False) if location_opt_choice == "2": _generate_from_location( - location, package_type, runtime, dependency_manager, output_dir, name, app_template, no_input + project_type, location, package_type, runtime, dependency_manager, output_dir, name, app_template, no_input ) else: if not pt_explicit: - click.echo("What package type would you like to use?") - click.echo("\t1 - Zip (artifact is a zip uploaded to S3)\t") - click.echo("\t2 - Image (artifact is an image uploaded to an ECR image repository)") - package_opt_choice = click.prompt("Package type", type=click.Choice(["1", "2"]), show_choices=False) - if package_opt_choice == "1": + if project_type == ProjectTypes.CDK: + cdk_language = _get_cdk_language(cdk_language) + click.echo("Example CDK project only have ZIP type (artifact is a zip uploaded to S3) now.") package_type = ZIP else: - package_type = IMAGE + click.echo("What package type would you like to use?") + click.echo("\t1 - Zip (artifact is a zip uploaded to S3)\t") + click.echo("\t2 - Image (artifact is an image uploaded to an ECR image repository)") + package_opt_choice = click.prompt("Package type", type=click.Choice(["1", "2"]), show_choices=False) + if package_opt_choice == "1": + package_type = ZIP + else: + package_type = IMAGE _generate_from_app_template( - location, package_type, runtime, base_image, dependency_manager, output_dir, name, app_template + project_type, + cdk_language, + location, + package_type, + runtime, + base_image, + dependency_manager, + output_dir, + name, + app_template, ) def _generate_from_location( - location, package_type, runtime, dependency_manager, output_dir, name, app_template, no_input + project_type, location, package_type, runtime, dependency_manager, output_dir, name, app_template, no_input ): location = click.prompt("\nTemplate location (git, mercurial, http(s), zip, path)", type=str) summary_msg = """ @@ -79,34 +103,45 @@ def _generate_from_location( location=location, output_dir=output_dir ) click.echo(summary_msg) - do_generate(location, package_type, runtime, dependency_manager, output_dir, name, no_input, None) + do_generate(project_type, location, package_type, runtime, dependency_manager, output_dir, name, no_input, None) # pylint: disable=too-many-statements def _generate_from_app_template( - location, package_type, runtime, base_image, dependency_manager, output_dir, name, app_template + project_type, + cdk_language, + location, + package_type, + runtime, + base_image, + dependency_manager, + output_dir, + name, + app_template, ): extra_context = None if package_type == IMAGE: base_image, runtime = _get_runtime_from_image(base_image) else: - runtime = _get_runtime(runtime) + runtime = _get_runtime(project_type, runtime) dependency_manager = _get_dependency_manager(dependency_manager, runtime) if not name: name = click.prompt("\nProject name", type=str, default="sam-app") templates = InitTemplates() if app_template is not None: location = templates.location_from_app_template( - package_type, runtime, base_image, dependency_manager, app_template + project_type, cdk_language, package_type, runtime, base_image, dependency_manager, app_template ) extra_context = {"project_name": name, "runtime": runtime} else: - location, app_template = templates.prompt_for_location(package_type, runtime, base_image, dependency_manager) + location, app_template = templates.prompt_for_location( + project_type, cdk_language, package_type, runtime, base_image, dependency_manager + ) extra_context = {"project_name": name, "runtime": runtime} # executing event_bridge logic if call is for Schema dynamic template is_dynamic_schemas_template = templates.is_dynamic_schemas_template( - package_type, app_template, runtime, base_image, dependency_manager + project_type, cdk_language, package_type, app_template, runtime, base_image, dependency_manager ) if is_dynamic_schemas_template: schemas_api_caller = get_schemas_api_caller() @@ -116,6 +151,7 @@ def _generate_from_app_template( no_input = True summary_msg = "" + cdk_language_msg = f"\n CDK application language: {cdk_language}\n" if cdk_language else "" if package_type == ZIP: summary_msg = f""" ----------------------- @@ -126,6 +162,7 @@ def _generate_from_app_template( Dependency Manager: {dependency_manager} Application Template: {app_template} Output Directory: {output_dir} + Project Type: {project_type.value} {cdk_language_msg} Next steps can be found in the README file at {output_dir}/{name}/README.md """ @@ -138,28 +175,57 @@ def _generate_from_app_template( Base Image: {base_image} Dependency Manager: {dependency_manager} Output Directory: {output_dir} + Project Type: {project_type.value} {cdk_language_msg} Next steps can be found in the README file at {output_dir}/{name}/README.md """ click.echo(summary_msg) - do_generate(location, package_type, runtime, dependency_manager, output_dir, name, no_input, extra_context) + do_generate( + project_type, location, package_type, runtime, dependency_manager, output_dir, name, no_input, extra_context + ) # executing event_bridge logic if call is for Schema dynamic template if is_dynamic_schemas_template: _package_schemas_code(runtime, schemas_api_caller, schema_template_details, output_dir, name, location) -def _get_runtime(runtime): +def _get_project_type(project_type): + if not project_type: + click.echo("Which type of project would you like to create?") + click.echo("\t1 - SAM\n\t2 - CDK") + project_type_choice = click.prompt("Choice", type=click.Choice(["1", "2"]), show_choices=False) + if project_type_choice == "1": + return ProjectTypes.CFN + return ProjectTypes.CDK + return project_type + + +def _get_cdk_language(cdk_language): + if not cdk_language: + choices = list(map(str, range(1, len(INIT_CDK_LANGUAGES) + 1))) + choice_num = 1 + click.echo("\nWhich language would you like to use for the CDK app?") + for r in INIT_CDK_LANGUAGES: + msg = "\t" + str(choice_num) + " - " + r + click.echo(msg) + choice_num = choice_num + 1 + choice = click.prompt("CDK Language", type=click.Choice(choices), show_choices=False) + cdk_language = INIT_CDK_LANGUAGES[int(choice) - 1] # zero index + return cdk_language + + +def _get_runtime(project_type, runtime): if not runtime: - choices = list(map(str, range(1, len(INIT_RUNTIMES) + 1))) + init_runtimes = CDK_INIT_RUNTIMES if project_type == ProjectTypes.CDK else INIT_RUNTIMES + choices = list(map(str, range(1, len(init_runtimes) + 1))) choice_num = 1 click.echo("\nWhich runtime would you like to use?") - for r in INIT_RUNTIMES: + for r in init_runtimes: msg = "\t" + str(choice_num) + " - " + r click.echo(msg) choice_num = choice_num + 1 choice = click.prompt("Runtime", type=click.Choice(choices), show_choices=False) - runtime = INIT_RUNTIMES[int(choice) - 1] # zero index + runtime = init_runtimes[int(choice) - 1] # zero index return runtime diff --git a/samcli/lib/iac/cdk/cloud_assembly.py b/samcli/lib/iac/cdk/cloud_assembly.py index de4ec337a1..2258b995b2 100644 --- a/samcli/lib/iac/cdk/cloud_assembly.py +++ b/samcli/lib/iac/cdk/cloud_assembly.py @@ -34,9 +34,7 @@ CDK_PATH_DELIMITER, CDK_PATH_METADATA_KEY, ) -from samcli.lib.iac.cdk.exceptions import ( - InvalidCloudAssemblyError, -) +from samcli.lib.iac.cdk.exceptions import InvalidCloudAssemblyError LOG = logging.getLogger(__name__) diff --git a/samcli/lib/init/__init__.py b/samcli/lib/init/__init__.py index 089d81f18e..42a90e5de1 100644 --- a/samcli/lib/init/__init__.py +++ b/samcli/lib/init/__init__.py @@ -10,16 +10,18 @@ from cookiecutter.exceptions import CookiecutterException, RepositoryNotFound, UnknownRepoType from cookiecutter.main import cookiecutter -from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING +from samcli.local.common.runtime_template import RUNTIME_DEP_TEMPLATE_MAPPING, CDK_RUNTIME_DEP_TEMPLATE_MAPPING from samcli.lib.utils.packagetype import ZIP from samcli.lib.utils import osutils from .exceptions import GenerateProjectFailedError, InvalidLocationError from .arbitrary_project import generate_non_cookiecutter_project +from ..iac.interface import ProjectTypes LOG = logging.getLogger(__name__) def generate_project( + project_type=None, location=None, package_type=None, runtime=None, @@ -37,6 +39,8 @@ def generate_project( Parameters ---------- + project_type: ProjectTypes + project type, CDK or CFN location: Path, optional Git, HTTP, Local path or Zip containing cookiecutter template (the default is None, which means no custom template) @@ -65,8 +69,11 @@ def generate_project( template = None - if runtime and package_type == ZIP: - for mapping in list(itertools.chain(*(RUNTIME_DEP_TEMPLATE_MAPPING.values()))): + if runtime and package_type == ZIP and not location: + runtime_dependency_template_mapping = ( + CDK_RUNTIME_DEP_TEMPLATE_MAPPING if project_type == ProjectTypes.CDK else RUNTIME_DEP_TEMPLATE_MAPPING + ) + for mapping in list(itertools.chain(*(runtime_dependency_template_mapping.values()))): if runtime in mapping["runtimes"] or any([r.startswith(runtime) for r in mapping["runtimes"]]): if not dependency_manager or dependency_manager == mapping["dependency_manager"]: template = mapping["init_location"] diff --git a/samcli/local/common/runtime_template.py b/samcli/local/common/runtime_template.py index 2924a5aa9f..265406a9d5 100644 --- a/samcli/local/common/runtime_template.py +++ b/samcli/local/common/runtime_template.py @@ -11,6 +11,25 @@ _templates = os.path.join(_init_path, "lib", "init", "templates") _lambda_images_templates = os.path.join(_init_path, "lib", "init", "image_templates") +CDK_RUNTIME_DEP_TEMPLATE_MAPPING = { + "python": [ + { + "runtimes": ["python3.8"], + "dependency_manager": "pip", + "init_location": os.path.join(_templates, "cookiecutter-aws-sam-hello-python"), + "build": True, + } + ], + "nodejs": [ + { + "runtimes": ["nodejs14.x"], + "dependency_manager": "npm", + "init_location": os.path.join(_templates, "cookiecutter-aws-sam-hello-nodejs"), + "build": True, + } + ], +} + # Note(TheSriram): The ordering of the runtimes list per language is based on the latest to oldest. RUNTIME_DEP_TEMPLATE_MAPPING = { "python": [ @@ -108,6 +127,13 @@ def get_local_lambda_images_location(mapping, runtime): ) ) +INIT_CDK_LANGUAGES = ["python", "javascript", "typescript"] + +CDK_INIT_RUNTIMES = [ + "nodejs14.x", + "python3.8", +] + # When adding new Lambda runtimes, please update SAM_RUNTIME_TO_SCHEMAS_CODE_LANG_MAPPING # Order here should be a the group of the latest versions of runtimes followed by runtime groups INIT_RUNTIMES = [ diff --git a/tests/integration/buildcmd/test_build_cmd.py b/tests/integration/buildcmd/test_build_cmd.py index 654441c71e..3765eb8a81 100644 --- a/tests/integration/buildcmd/test_build_cmd.py +++ b/tests/integration/buildcmd/test_build_cmd.py @@ -1821,9 +1821,9 @@ def test_nested_build(self, use_container, cached, parallel): stack_paths = ["", "LocalNestedStack"] if not SKIP_DOCKER_TESTS: self._verify_build( - [], # there is no function artifact dirs to check + [], stack_paths, - command_result, + command_result, # there is no function artifact dirs to check ) overrides = self._make_parameter_override_arg(overrides) @@ -1975,8 +1975,8 @@ def test_functions_layers_with_s3_codeuri(self): if not SKIP_DOCKER_TESTS: self._verify_build( ["ServerlessFunction", "LambdaFunction"], - [""], # there is only one stack - command_result, + [""], + command_result, # there is only one stack ) # these two functions are buildable and `sam build` would build it. # but since the two functions both depends on layers with s3 uri, @@ -2011,6 +2011,6 @@ def test_functions_layers_with_s3_codeuri(self): # which are self._verify_build( [], - [""], # there is only one stack - command_result, + [""], + command_result, # there is only one stack ) diff --git a/tests/unit/commands/_utils/test_options.py b/tests/unit/commands/_utils/test_options.py index 592972efb6..2660d1451a 100644 --- a/tests/unit/commands/_utils/test_options.py +++ b/tests/unit/commands/_utils/test_options.py @@ -231,9 +231,9 @@ def test_artifact_different_from_required_option(self, template_artifacts_mock): mock_params = MagicMock() mock_params.get = MagicMock( side_effect=[ - MagicMock(), # mock_params.get("t") - False, # mock_params.get("resolve_s3") - ] + MagicMock(), + False, + ] # mock_params.get("t") # mock_params.get("resolve_s3") ) mock_default_map = MagicMock() mock_default_map.get = MagicMock(return_value=False) diff --git a/tests/unit/commands/init/test_cli.py b/tests/unit/commands/init/test_cli.py index d80b3022a7..874819003f 100644 --- a/tests/unit/commands/init/test_cli.py +++ b/tests/unit/commands/init/test_cli.py @@ -8,6 +8,7 @@ from samcli.commands.init.init_templates import InitTemplates from samcli.commands.init import cli as init_cmd from samcli.commands.init import do_cli as init_cli +from samcli.lib.iac.interface import ProjectTypes from samcli.lib.init import GenerateProjectFailedError from samcli.commands.exceptions import UserException from samcli.lib.utils.packagetype import IMAGE, ZIP @@ -60,11 +61,14 @@ def test_init_cli(self, generate_project_patch, sd_mock): no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN we should receive no errors generate_project_patch.assert_called_once_with( # need to change the location validation check + ProjectTypes.CFN, ANY, ZIP, self.runtime, @@ -95,11 +99,14 @@ def test_init_image_cli(self, generate_project_patch, sd_mock): no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN we should receive no errors generate_project_patch.assert_called_once_with( # need to change the location validation check + ProjectTypes.CFN, ANY, IMAGE, "nodejs12.x", @@ -130,11 +137,14 @@ def test_init_image_java_cli(self, generate_project_patch, sd_mock): no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN we should receive no errors generate_project_patch.assert_called_once_with( # need to change the location validation check + ProjectTypes.CFN, ANY, IMAGE, "java11", @@ -165,6 +175,8 @@ def test_init_fails_invalid_template(self, sd_mock): no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @@ -187,6 +199,8 @@ def test_init_fails_invalid_dep_mgr(self, sd_mock): no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @@ -215,10 +229,18 @@ def test_init_cli_generate_project_fails(self, generate_project_patch, sd_mock): no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) generate_project_patch.assert_called_with( - self.location, self.runtime, self.dependency_manager, self.output_dir, self.name, self.no_input + ProjectTypes.CFN, + self.location, + self.runtime, + self.dependency_manager, + self.output_dir, + self.name, + self.no_input, ) @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") @@ -247,10 +269,18 @@ def test_init_cli_generate_project_image_fails(self, generate_project_patch, sd_ no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) generate_project_patch.assert_called_with( - self.location, self.runtime, self.dependency_manager, self.output_dir, self.name, self.no_input + ProjectTypes.CFN, + self.location, + self.runtime, + self.dependency_manager, + self.output_dir, + self.name, + self.no_input, ) @patch("samcli.commands.init.init_generator.generate_project") @@ -272,11 +302,21 @@ def test_init_cli_with_extra_context_parameter_not_passed(self, generate_project no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN we should receive no errors generate_project_patch.assert_called_once_with( - ANY, ZIP, self.runtime, self.dependency_manager, ".", self.name, True, self.extra_context_as_json + ProjectTypes.CFN, + ANY, + ZIP, + self.runtime, + self.dependency_manager, + ".", + self.name, + True, + self.extra_context_as_json, ) @patch("samcli.commands.init.init_generator.generate_project") @@ -298,10 +338,13 @@ def test_init_cli_with_extra_context_parameter_passed(self, generate_project_pat no_input=self.no_input, extra_context='{"schema_name":"events", "schema_type":"aws"}', auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN we should receive no errors and right extra_context should be passed generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, ZIP, self.runtime, @@ -331,10 +374,13 @@ def test_init_cli_with_extra_context_not_overriding_default_parameter(self, gene no_input=self.no_input, extra_context='{"project_name": "my_project", "runtime": "java8", "schema_name":"events", "schema_type": "aws"}', auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN extra_context should have not overridden default_parameters(name, runtime) generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, ZIP, self.runtime, @@ -364,6 +410,8 @@ def test_init_cli_with_extra_context_input_as_wrong_json_raises_exception(self): no_input=self.no_input, extra_context='{"project_name", "my_project", "runtime": "java8", "schema_name":"events", "schema_type": "aws"}', auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) @patch("samcli.commands.init.init_generator.generate_project") @@ -385,10 +433,13 @@ def test_init_cli_must_set_default_context_when_location_is_provided(self, gener no_input=None, extra_context='{"schema_name":"events", "schema_type": "aws"}', auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN should set default parameter(name, runtime) as extra_context generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, "custom location", ZIP, "java8", @@ -418,10 +469,13 @@ def test_init_cli_must_only_set_passed_project_name_when_location_is_provided(se no_input=None, extra_context='{"schema_name":"events", "schema_type": "aws"}', auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN extra_context should be without runtime generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, "custom location", ZIP, None, @@ -451,10 +505,13 @@ def test_init_cli_must_only_set_passed_runtime_when_location_is_provided(self, g no_input=None, extra_context='{"schema_name":"events", "schema_type": "aws"}', auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN extra_context should be without name generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, "custom location", ZIP, "java8", @@ -486,10 +543,13 @@ def test_init_cli_with_extra_context_parameter_passed_as_escaped(self, generate_ extra_context='{\"schema_name\":\"events\", \"schema_type\":\"aws\"}', # fmt: on auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) # THEN we should receive no errors and right extra_context should be passed generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, ZIP, self.runtime, @@ -558,6 +618,7 @@ def test_init_cli_int_with_event_bridge_app_template( schemas_api_caller_mock.return_value.download_source_code_binding.return_value = "result.zip" # WHEN the user follows interactive init prompts + # 1: Project Type: SAM # 1: AWS Quick Start Templates # 5: Java Runtime # 1: dependency manager maven @@ -570,6 +631,7 @@ def test_init_cli_int_with_event_bridge_app_template( user_input = """ 1 1 +1 5 1 test-project @@ -584,6 +646,7 @@ def test_init_cli_int_with_event_bridge_app_template( result = runner.invoke(init_cmd, input=user_input) self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, ZIP, "java11", @@ -625,6 +688,7 @@ def test_init_cli_int_with_image_app_template( # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 1: AWS Quick Start Templates # 2: Package type - Image # 13: Java8 base image @@ -633,6 +697,7 @@ def test_init_cli_int_with_image_app_template( user_input = """ 1 +1 2 13 1 @@ -642,6 +707,7 @@ def test_init_cli_int_with_image_app_template( result = runner.invoke(init_cmd, input=user_input) generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, IMAGE, "java8", @@ -652,6 +718,53 @@ def test_init_cli_int_with_image_app_template( {"project_name": "test-project", "runtime": "java8"}, ) + @patch.object(InitTemplates, "__init__", MockInitTemplates.__init__) + @patch("samcli.commands.init.init_templates.InitTemplates._init_options_from_manifest") + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_int_with_cdk_zip_app_template( + self, + generate_project_patch, + init_options_from_manifest_mock, + ): + init_options_from_manifest_mock.return_value = [ + { + "directory": "cdk-python/nodejs14.x/cookiecutter-aws-sam-hello-nodejs", + "displayName": "Hello World Example", + "dependencyManager": "npm", + "appTemplate": "hello-world", + } + ] + + # WHEN the user follows interactive init prompts + + # 2: Project type: CDK + # 1: AWS Quick Start Templates + # 1: CDK language - Python + # 1: Runtime - nodejs14.x + # test-project: response to name + + user_input = """ +2 +1 +1 +1 +test-project + """ + runner = CliRunner() + result = runner.invoke(init_cmd, input=user_input) + + generate_project_patch.assert_called_once_with( + ProjectTypes.CDK, + ANY, + ZIP, + "nodejs14.x", + "npm", + ".", + "test-project", + True, + {"project_name": "test-project", "runtime": "nodejs14.x"}, + ) + @patch.object(InitTemplates, "__init__", MockInitTemplates.__init__) @patch("samcli.commands.init.init_templates.InitTemplates._init_options_from_manifest") @patch("samcli.lib.schemas.schemas_aws_config.Session") @@ -712,6 +825,7 @@ def test_init_cli_int_with_event_bridge_app_template_and_aws_configuration( schemas_api_caller_mock.return_value.download_source_code_binding.return_value = "result.zip" # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 1: AWS Quick Start Templates # 5: Java Runtime # 1: dependency manager maven @@ -726,6 +840,7 @@ def test_init_cli_int_with_event_bridge_app_template_and_aws_configuration( user_input = """ 1 1 +1 5 1 test-project @@ -743,6 +858,7 @@ def test_init_cli_int_with_event_bridge_app_template_and_aws_configuration( self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, ZIP, "java11", @@ -795,6 +911,7 @@ def test_init_cli_int_with_event_bridge_app_template_and_aws_configuration_with_ ) # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 1: AWS Quick Start Templates # 5: Java Runtime # 1: dependency manager maven @@ -809,6 +926,7 @@ def test_init_cli_int_with_event_bridge_app_template_and_aws_configuration_with_ user_input = """ 1 1 +1 5 1 test-project @@ -887,6 +1005,7 @@ def test_init_cli_int_with_download_manager_raises_exception( ) # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 1: AWS Quick Start Templates # 5: Java Runtime # 1: dependency manager maven @@ -899,6 +1018,7 @@ def test_init_cli_int_with_download_manager_raises_exception( user_input = """ 1 1 +1 5 1 test-project @@ -913,6 +1033,7 @@ def test_init_cli_int_with_download_manager_raises_exception( result = runner.invoke(init_cmd, input=user_input) self.assertTrue(result.exception) generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, ZIP, "java11", @@ -988,6 +1109,7 @@ def test_init_cli_int_with_schemas_details_raises_exception( {"Error": {"Code": "ConflictException", "Message": "ConflictException"}}, "operation" ) # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 1: AWS Quick Start Templates # 5: Java Runtime # 1: dependency manager maven @@ -1000,6 +1122,7 @@ def test_init_cli_int_with_schemas_details_raises_exception( user_input = """ 1 1 +1 5 1 test-project @@ -1036,10 +1159,13 @@ def test_init_passes_dynamic_event_bridge_template(self, generate_project_patch, no_input=self.no_input, extra_context=None, auto_clone=False, + project_type=ProjectTypes.CFN, + cdk_language=None, ) generate_project_patch.assert_called_once_with( # need to change the location validation check + ProjectTypes.CFN, ANY, ZIP, self.runtime, @@ -1055,9 +1181,11 @@ def test_init_passes_dynamic_event_bridge_template(self, generate_project_patch, def test_init_cli_int_from_location(self, generate_project_patch, sd_mock): # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 2: selecting custom location # foo: the "location" user_input = """ +1 2 foo """ @@ -1069,6 +1197,39 @@ def test_init_cli_int_from_location(self, generate_project_patch, sd_mock): self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( # need to change the location validation check + ProjectTypes.CFN, + "foo", + ZIP, + None, + None, + ".", + None, + False, + None, + ) + + @patch("samcli.commands.init.init_templates.InitTemplates._shared_dir_check") + @patch("samcli.commands.init.init_generator.generate_project") + def test_init_cli_int_cdk_project_from_location(self, generate_project_patch, sd_mock): + # WHEN the user follows interactive init prompts + + # 2: Project type: CDK + # 2: selecting custom location + # foo: the "location" + user_input = """ +2 +2 +foo + """ + + runner = CliRunner() + result = runner.invoke(init_cmd, input=user_input) + + # THEN we should receive no errors + self.assertFalse(result.exception) + generate_project_patch.assert_called_once_with( + # need to change the location validation check + ProjectTypes.CDK, "foo", ZIP, None, @@ -1084,10 +1245,12 @@ def test_init_cli_int_from_location(self, generate_project_patch, sd_mock): def test_init_cli_no_package_type(self, generate_project_patch, sd_mock): # WHEN the user follows interactive init prompts + # 1: Project type: SAM # 1: selecting template source # 2s: selecting package type user_input = """ 1 +1 2 1 """ @@ -1106,6 +1269,7 @@ def test_init_cli_no_package_type(self, generate_project_patch, sd_mock): # THEN we should receive no errors self.assertFalse(result.exception) generate_project_patch.assert_called_once_with( + ProjectTypes.CFN, ANY, IMAGE, "python3.8", diff --git a/tests/unit/commands/init/test_templates.py b/tests/unit/commands/init/test_templates.py index b422b5dbf0..869fe0b907 100644 --- a/tests/unit/commands/init/test_templates.py +++ b/tests/unit/commands/init/test_templates.py @@ -5,6 +5,8 @@ from unittest.mock import mock_open, patch, PropertyMock, MagicMock from re import search from unittest import TestCase + +from samcli.lib.iac.interface import ProjectTypes from samcli.lib.utils.packagetype import IMAGE, ZIP from pathlib import Path @@ -37,7 +39,9 @@ def test_location_from_app_template_zip(self, subprocess_mock, git_exec_mock, sd with patch("samcli.cli.global_config.GlobalConfig.config_dir", new_callable=PropertyMock) as mock_cfg: mock_cfg.return_value = "/tmp/test-sam" with patch("samcli.commands.init.init_templates.open", m): - location = it.location_from_app_template(ZIP, "ruby2.5", None, "bundler", "hello-world") + location = it.location_from_app_template( + ProjectTypes.CFN, None, ZIP, "ruby2.5", None, "bundler", "hello-world" + ) self.assertTrue(search("mock-ruby-template", location)) @patch("subprocess.check_output") @@ -65,7 +69,7 @@ def test_location_from_app_template_image(self, subprocess_mock, git_exec_mock, mock_cfg.return_value = "/tmp/test-sam" with patch("samcli.commands.init.init_templates.open", m): location = it.location_from_app_template( - IMAGE, None, "ruby2.5-image", "bundler", "hello-world-lambda-image" + ProjectTypes.CFN, None, IMAGE, None, "ruby2.5-image", "bundler", "hello-world-lambda-image" ) self.assertTrue(search("mock-ruby-image-template", location)) @@ -79,7 +83,7 @@ def test_fallback_options(self, git_exec_mock, prompt_mock, sd_mock): mock_sub.side_effect = OSError("Fail") mock_cfg.return_value = "/tmp/test-sam" it = InitTemplates(True) - location, app_template = it.prompt_for_location(ZIP, "ruby2.5", None, "bundler") + location, app_template = it.prompt_for_location(ProjectTypes.CFN, None, ZIP, "ruby2.5", None, "bundler") self.assertTrue(search("cookiecutter-aws-sam-hello-ruby", location)) self.assertEqual("hello-world", app_template) @@ -93,7 +97,7 @@ def test_fallback_process_error(self, git_exec_mock, prompt_mock, sd_mock): mock_sub.side_effect = subprocess.CalledProcessError("fail", "fail", "not found".encode("utf-8")) mock_cfg.return_value = "/tmp/test-sam" it = InitTemplates(True) - location, app_template = it.prompt_for_location(ZIP, "ruby2.5", None, "bundler") + location, app_template = it.prompt_for_location(ProjectTypes.CFN, None, ZIP, "ruby2.5", None, "bundler") self.assertTrue(search("cookiecutter-aws-sam-hello-ruby", location)) self.assertEqual("hello-world", app_template) diff --git a/tests/unit/commands/samconfig/test_samconfig.py b/tests/unit/commands/samconfig/test_samconfig.py index 7e2fbb5393..4b27b01827 100644 --- a/tests/unit/commands/samconfig/test_samconfig.py +++ b/tests/unit/commands/samconfig/test_samconfig.py @@ -16,6 +16,7 @@ from unittest.mock import patch, ANY, Mock, MagicMock import logging +from samcli.lib.iac.interface import ProjectTypes from samcli.lib.utils.packagetype import ZIP, IMAGE LOG = logging.getLogger() @@ -78,6 +79,8 @@ def test_init(self, do_cli_mock): "apptemplate", True, '{"key": "value", "key2": "value2"}', + None, + None, ) @patch("samcli.commands.validate.validate.do_cli") From b4e66f4d39cda6551035ebe19029386f94749975 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 29 Jun 2021 13:43:20 -0700 Subject: [PATCH 03/38] Trigger appveyor From 6b480998e73abbe84c7bd905533406ceba1a7d6e Mon Sep 17 00:00:00 2001 From: _sam <3804518+aahung@users.noreply.github.com> Date: Thu, 10 Jun 2021 10:22:14 -0700 Subject: [PATCH 04/38] ci: Pin boto3-stubs to 1.17.90 due to a bug in 1.17.91 (#2942) --- requirements/dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 6c494f8ec3..f072365258 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,8 @@ pytest-cov==2.10.1 # mypy adds new rules in new minor versions, which could cause our PR check to fail # here we fix its version and upgrade it manually in the future mypy==0.790 -boto3-stubs[essential]~=1.14 +# 1.17.91 has a bug, https://github.com/vemel/mypy_boto3_builder/pull/82 +boto3-stubs[essential]==1.17.90.post1 # Test requirements pytest==6.1.1 From 880b3ef70ca45b780118558a954842d3b72e0d60 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 29 Jun 2021 14:44:31 -0700 Subject: [PATCH 05/38] black reformat --- samcli/commands/init/init_templates.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samcli/commands/init/init_templates.py b/samcli/commands/init/init_templates.py index f1ac5bf64e..4cd025593a 100644 --- a/samcli/commands/init/init_templates.py +++ b/samcli/commands/init/init_templates.py @@ -29,6 +29,7 @@ class InvalidInitTemplateError(UserException): pass + class CDKProjectInvalidInitConfigError(UserException): pass From 2cef205cf30d4e5ce7591b3e159934a2716b40b3 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Tue, 29 Jun 2021 15:08:31 -0700 Subject: [PATCH 06/38] Cdk support package and deploy fix (#2996) * Fix --resolve-s3 --s3-bucket validation under guided flow * Fix package resource assets --- samcli/commands/package/validations.py | 5 +++-- samcli/lib/iac/interface.py | 6 ++++++ samcli/lib/package/packageable_resources.py | 4 +++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/samcli/commands/package/validations.py b/samcli/commands/package/validations.py index 18f14411af..27648970f3 100644 --- a/samcli/commands/package/validations.py +++ b/samcli/commands/package/validations.py @@ -30,12 +30,13 @@ def wrapped(*args, **kwargs): # NOTE(sriram-mv): Both params and default_map need to be checked, as the option can be either be # passed in directly or through configuration file. # If passed in through configuration file, default_map is loaded with those values. + guided = ctx.params.get("guided", False) or ctx.params.get("g", False) resolve_s3_provided = ctx.params.get("resolve_s3", False) or ctx.default_map.get("resolve_s3", False) s3_bucket_provided = ctx.params.get("s3_bucket", False) or ctx.default_map.get("s3_bucket", False) either_required = stack.has_assets_of_package_type(ZIP) if stack is not None else False - if s3_bucket_provided and resolve_s3_provided: + if not guided and s3_bucket_provided and resolve_s3_provided: raise PackageResolveS3AndS3SetError() - if either_required and not s3_bucket_provided and not resolve_s3_provided: + if not guided and either_required and not s3_bucket_provided and not resolve_s3_provided: raise PackageResolveS3AndS3NotSetError() return func(*args, **kwargs) diff --git a/samcli/lib/iac/interface.py b/samcli/lib/iac/interface.py index 197a3b084b..5a8c2ea27f 100644 --- a/samcli/lib/iac/interface.py +++ b/samcli/lib/iac/interface.py @@ -663,6 +663,12 @@ def find_function_resources_of_package_type(self, package_type: str) -> List[Res _function_resources.append(resource) return _function_resources + def get_overrideable_parameters(self): + """ + Return a dict of parameters that are override-able, i.e. not added by iac + """ + return {key: val for key, val in self.get("Parameters", {}).items() if not val.added_by_iac} + def as_dict(self): """ return the stack as a dict for JSON serialization diff --git a/samcli/lib/package/packageable_resources.py b/samcli/lib/package/packageable_resources.py index 4dbf331c4e..c25114f6c9 100644 --- a/samcli/lib/package/packageable_resources.py +++ b/samcli/lib/package/packageable_resources.py @@ -108,7 +108,9 @@ def export(self, resource, parent_dir): if not resource.is_packageable(): return - # we can safely assume resource.assets contains at lease one asset + if not resource.assets: + return + # resource.assets contains at lease one asset asset = resource.assets[0] if not asset.source_path and not self.PACKAGE_NULL_PROPERTY: From 3aea77610f2889fd3132dbb45eb989f7a9d82084 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 29 Jun 2021 20:21:05 -0700 Subject: [PATCH 07/38] Add debug --- tests/integration/buildcmd/build_integ_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 5753f8836f..754ae74a52 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -74,7 +74,7 @@ def get_command_list( build_image=None, ): - command_list = [self.cmd, "build"] + command_list = [self.cmd, "build --debug"] if function_identifier: command_list += [function_identifier] From 073d4c58511ec5344f584593e076b60519d66e41 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 29 Jun 2021 21:03:11 -0700 Subject: [PATCH 08/38] Trigger test with debug --- tests/integration/buildcmd/build_integ_base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/buildcmd/build_integ_base.py b/tests/integration/buildcmd/build_integ_base.py index 754ae74a52..8255dc3c78 100644 --- a/tests/integration/buildcmd/build_integ_base.py +++ b/tests/integration/buildcmd/build_integ_base.py @@ -74,12 +74,12 @@ def get_command_list( build_image=None, ): - command_list = [self.cmd, "build --debug"] + command_list = [self.cmd, "build"] if function_identifier: command_list += [function_identifier] - command_list += ["-t", self.template_path] + command_list += ["-t", self.template_path, "--debug"] if parameter_overrides: command_list += ["--parameter-overrides", self._make_parameter_override_arg(parameter_overrides)] From 3d3403db4d537dd7f207ebbe5416a2a283cd58b7 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar Date: Wed, 30 Jun 2021 00:00:43 -0700 Subject: [PATCH 09/38] restart docker service in linux --- appveyor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/appveyor.yml b/appveyor.yml index fb1a652138..baa056e5d2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -127,6 +127,7 @@ for: - sh: "PATH=$JAVA_HOME/bin:$PATH" - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - sh: "rvm use 2.5" + - sh: "sudo service docker restart" - sh: "docker info" # Install latest gradle From 3ce48dc8c84d136453eb91bbd1b46da9a967de7b Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar Date: Wed, 30 Jun 2021 00:38:41 -0700 Subject: [PATCH 10/38] revert - restart docker service in linux --- appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index baa056e5d2..fb1a652138 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -127,7 +127,6 @@ for: - sh: "PATH=$JAVA_HOME/bin:$PATH" - sh: "source ${HOME}/venv${PYTHON_VERSION}/bin/activate" - sh: "rvm use 2.5" - - sh: "sudo service docker restart" - sh: "docker info" # Install latest gradle From e948298f1279c973fb8b596d39942afb18a32626 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 30 Jun 2021 10:38:29 -0700 Subject: [PATCH 11/38] Update appveyor.yml to log into ECR --- appveyor.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index fb1a652138..0124291ce0 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -183,10 +183,9 @@ for: - "mypy setup.py samcli tests" - "pytest -n 4 tests/functional" - # Runs only in Linux, logging docker hub when running canary and docker cred is available - sh: " - if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; - then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; + if [[ -n $BY_CANARY ]]; + then echo Logging in ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; fi" - sh: "pytest -vv tests/integration" - sh: "pytest -vv tests/regression" From 8a8248a48a29ea28b1b08cd97e52587f24008a5e Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 30 Jun 2021 11:44:17 -0700 Subject: [PATCH 12/38] Revert "Update appveyor.yml to log into ECR" This reverts commit e948298f1279c973fb8b596d39942afb18a32626. --- appveyor.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0124291ce0..fb1a652138 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -183,9 +183,10 @@ for: - "mypy setup.py samcli tests" - "pytest -n 4 tests/functional" + # Runs only in Linux, logging docker hub when running canary and docker cred is available - sh: " - if [[ -n $BY_CANARY ]]; - then echo Logging in ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; + then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; fi" - sh: "pytest -vv tests/integration" - sh: "pytest -vv tests/regression" From 72f697b1088df8ba3f4dbb52d27c483357e8bebc Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 30 Jun 2021 11:52:35 -0700 Subject: [PATCH 13/38] Update appveyor.yml to log into Public ECR --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index fb1a652138..5c3c53ccfe 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -188,6 +188,10 @@ for: if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; fi" + - sh: " + if [[ -n $BY_CANARY ]]; + then echo Logging in Public ECR; aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws; + fi" - sh: "pytest -vv tests/integration" - sh: "pytest -vv tests/regression" - sh: "black --check setup.py tests samcli" From 6220dd0fab21e4e55f0dcead4de5ff17bc7693d7 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 30 Jun 2021 13:26:31 -0700 Subject: [PATCH 14/38] Update appveyor.yml to explicitly specify server for logging in dockerhub --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5c3c53ccfe..d5f9c6e044 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -186,7 +186,7 @@ for: # Runs only in Linux, logging docker hub when running canary and docker cred is available - sh: " if [[ -n $BY_CANARY ]] && [[ -n $DOCKER_USER ]] && [[ -n $DOCKER_PASS ]]; - then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin; + then echo Logging in Docker Hub; echo $DOCKER_PASS | docker login --username $DOCKER_USER --password-stdin registry-1.docker.io; fi" - sh: " if [[ -n $BY_CANARY ]]; From 921a590995259d9cbb8926071900594b78722d56 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Mon, 5 Jul 2021 08:15:08 -0700 Subject: [PATCH 15/38] Disable python3.7, 3.6 to run integ test without pull limitation --- appveyor.yml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index d5f9c6e044..724081f1c3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,26 +9,26 @@ environment: matrix: - - PYTHON_HOME: "C:\\Python36-x64" - PYTHON_VERSION: '3.6' - PYTHON_ARCH: '64' - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_37_PIP: 1 - INSTALL_PY_38_PIP: 1 - AWS_S3: 'AWS_S3_36' - AWS_ECR: 'AWS_ECR_36' - APPVEYOR_CONSOLE_DISABLE_PTY: true - - - PYTHON_HOME: "C:\\Python37-x64" - PYTHON_VERSION: '3.7' - PYTHON_ARCH: '64' - RUN_SMOKE: 1 - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_36_PIP: 1 - INSTALL_PY_38_PIP: 1 - AWS_S3: 'AWS_S3_37' - AWS_ECR: 'AWS_ECR_37' - APPVEYOR_CONSOLE_DISABLE_PTY: true + # - PYTHON_HOME: "C:\\Python36-x64" + # PYTHON_VERSION: '3.6' + # PYTHON_ARCH: '64' + # NOSE_PARAMETERIZED_NO_WARN: 1 + # INSTALL_PY_37_PIP: 1 + # INSTALL_PY_38_PIP: 1 + # AWS_S3: 'AWS_S3_36' + # AWS_ECR: 'AWS_ECR_36' + # APPVEYOR_CONSOLE_DISABLE_PTY: true + + # - PYTHON_HOME: "C:\\Python37-x64" + # PYTHON_VERSION: '3.7' + # PYTHON_ARCH: '64' + # RUN_SMOKE: 1 + # NOSE_PARAMETERIZED_NO_WARN: 1 + # INSTALL_PY_36_PIP: 1 + # INSTALL_PY_38_PIP: 1 + # AWS_S3: 'AWS_S3_37' + # AWS_ECR: 'AWS_ECR_37' + # APPVEYOR_CONSOLE_DISABLE_PTY: true - PYTHON_HOME: "C:\\Python38-x64" PYTHON_VERSION: '3.8' From 41a03877c0716e41f0362c5bea501ec617cd2981 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Mon, 5 Jul 2021 21:34:59 -0700 Subject: [PATCH 16/38] fix rapid version regex --- samcli/local/docker/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/local/docker/manager.py b/samcli/local/docker/manager.py index dc684b9863..6ff722d374 100644 --- a/samcli/local/docker/manager.py +++ b/samcli/local/docker/manager.py @@ -197,7 +197,7 @@ def _is_rapid_image(image_name: str) -> bool: : return bool: True, if the image name ends with rapid-$SAM_CLI_VERSION. False, otherwise """ - if not re.search(r":rapid-\d+\.\d+.\d+$", image_name): + if not re.search(r":rapid-\d+\.\d+\.\w+$", image_name): return False return True From c2e22be6a63aed121f5f956c7531d4907abd055e Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 6 Jul 2021 09:16:59 -0700 Subject: [PATCH 17/38] Update regex --- samcli/local/docker/manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/local/docker/manager.py b/samcli/local/docker/manager.py index 6ff722d374..a2d692830b 100644 --- a/samcli/local/docker/manager.py +++ b/samcli/local/docker/manager.py @@ -197,7 +197,7 @@ def _is_rapid_image(image_name: str) -> bool: : return bool: True, if the image name ends with rapid-$SAM_CLI_VERSION. False, otherwise """ - if not re.search(r":rapid-\d+\.\d+\.\w+$", image_name): + if not re.search(r":rapid-\d+\.\d+\.[\w\.]+$", image_name): return False return True From 1393aae36058da7b883a724dca1b99f9151b2546 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 6 Jul 2021 16:11:11 -0700 Subject: [PATCH 18/38] fix integ test options --- .../init/schemas/schemas_test_data_setup.py | 2 ++ .../init/schemas/test_init_with_schemas_command.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/integration/init/schemas/schemas_test_data_setup.py b/tests/integration/init/schemas/schemas_test_data_setup.py index cb0c353649..4b40b55645 100644 --- a/tests/integration/init/schemas/schemas_test_data_setup.py +++ b/tests/integration/init/schemas/schemas_test_data_setup.py @@ -33,6 +33,7 @@ def setUpClass(cls): setup_schema_data_for_pagination("test-pagination", schemas_client) setup_non_partner_schema_data("other-schema", schemas_client) # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 13: Java runtime @@ -44,6 +45,7 @@ def setUpClass(cls): user_input = """ 1 1 +1 13 1 eb-app-maven diff --git a/tests/integration/init/schemas/test_init_with_schemas_command.py b/tests/integration/init/schemas/test_init_with_schemas_command.py index ad2f2ecb87..2efa40635f 100644 --- a/tests/integration/init/schemas/test_init_with_schemas_command.py +++ b/tests/integration/init/schemas/test_init_with_schemas_command.py @@ -18,6 +18,7 @@ class TestBasicInitWithEventBridgeCommand(SchemaTestDataSetup): def test_init_interactive_with_event_bridge_app_aws_registry(self): # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 14: Java runtime @@ -31,6 +32,7 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): user_input = """ 1 1 +1 14 1 eb-app-maven @@ -54,6 +56,7 @@ def test_init_interactive_with_event_bridge_app_aws_registry(self): def test_init_interactive_with_event_bridge_app_partner_registry(self): # setup schema data # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 14: Java runtime @@ -67,6 +70,7 @@ def test_init_interactive_with_event_bridge_app_partner_registry(self): user_input = """ 1 1 +1 14 1 eb-app-maven @@ -101,6 +105,7 @@ def test_init_interactive_with_event_bridge_app_partner_registry(self): def test_init_interactive_with_event_bridge_app_pagination(self): # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 14: Java Runtime @@ -116,6 +121,7 @@ def test_init_interactive_with_event_bridge_app_pagination(self): user_input = """ 1 1 +1 14 1 eb-app-maven @@ -141,6 +147,7 @@ def test_init_interactive_with_event_bridge_app_pagination(self): def test_init_interactive_with_event_bridge_app_customer_registry(self): # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 14: Java Runtime @@ -154,6 +161,7 @@ def test_init_interactive_with_event_bridge_app_customer_registry(self): user_input = """ 1 1 +1 14 1 eb-app-maven @@ -188,6 +196,7 @@ def test_init_interactive_with_event_bridge_app_customer_registry(self): def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 9: Python 3.7 @@ -200,6 +209,7 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): user_input = """ 1 1 +1 9 eb-app-python37 3 @@ -220,6 +230,7 @@ def test_init_interactive_with_event_bridge_app_aws_schemas_python(self): def test_init_interactive_with_event_bridge_app_non_default_profile_selection(self): self._init_custom_config("mynewprofile", "us-west-2") # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Packagetype # 9: Python 3.7 @@ -234,6 +245,7 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se user_input = """ 1 1 +1 9 eb-app-python37 3 @@ -258,6 +270,7 @@ def test_init_interactive_with_event_bridge_app_non_default_profile_selection(se def test_init_interactive_with_event_bridge_app_non_supported_schemas_region(self): self._init_custom_config("default", "cn-north-1") # WHEN the user follows interactive init prompts + # 1: SAM type project # 1: AWS Quick Start Templates # 1: Zip Pacakgetype # 9: Python 3.7 @@ -270,6 +283,7 @@ def test_init_interactive_with_event_bridge_app_non_supported_schemas_region(sel user_input = """ 1 1 +1 9 eb-app-python37 3 From 859a0a5fa8efcf7d28c0f6b0ce45a415e4fa80d8 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar Date: Tue, 6 Jul 2021 19:31:34 -0700 Subject: [PATCH 19/38] fix parsing the Lambda Function Image Uri --- samcli/lib/providers/sam_function_provider.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/samcli/lib/providers/sam_function_provider.py b/samcli/lib/providers/sam_function_provider.py index 1d95a20059..20a7951c1b 100644 --- a/samcli/lib/providers/sam_function_provider.py +++ b/samcli/lib/providers/sam_function_provider.py @@ -319,7 +319,11 @@ def _convert_lambda_function_resource( for asset in assets: if isinstance(asset, ImageAsset) and asset.source_property.startswith("Code"): image_asset = asset - function_image_uri = image_asset.source_local_image + code = image_asset.source_local_image + if isinstance(code, str): + function_image_uri = cast(Optional[str], code) + else: + function_image_uri = cast(Optional[str], code.get("ImageUri", None)) break imageuri = function_image_uri or SamFunctionProvider._extract_lambda_function_imageuri( From e77d692426adc8ca480afecd7332d0e1631274f3 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 6 Jul 2021 21:32:11 -0700 Subject: [PATCH 20/38] try fixing another integ test issue --- samcli/lib/providers/sam_function_provider.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/samcli/lib/providers/sam_function_provider.py b/samcli/lib/providers/sam_function_provider.py index 20a7951c1b..e53782fead 100644 --- a/samcli/lib/providers/sam_function_provider.py +++ b/samcli/lib/providers/sam_function_provider.py @@ -227,7 +227,11 @@ def _convert_sam_function_resource( for asset in assets: if isinstance(asset, ImageAsset) and asset.source_property == "ImageUri": image_asset = asset - function_image_uri = image_asset.source_local_image + code = image_asset.source_local_image + if isinstance(code, str): + function_image_uri = cast(Optional[str], code) + else: + function_image_uri = cast(Optional[str], code.get("ImageUri", None)) break imageuri = function_image_uri or SamFunctionProvider._extract_sam_function_imageuri( resource_properties, "ImageUri" From 542fae4a6e39558ce7267af3247094c39db03db2 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar Date: Wed, 7 Jul 2021 02:24:39 -0700 Subject: [PATCH 21/38] resolve the resources assets --- .../intrinsic_property_resolver.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py b/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py index 82d7864332..338e616549 100644 --- a/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py +++ b/samcli/lib/intrinsic_resolver/intrinsic_property_resolver.py @@ -8,6 +8,7 @@ import re from collections import OrderedDict +from samcli.lib.iac.interface import Resource, S3Asset, ImageAsset from samcli.lib.intrinsic_resolver.invalid_intrinsic_validation import ( verify_intrinsic_type_list, verify_non_null, @@ -249,11 +250,25 @@ def resolve_template(self, ignore_errors=False): if self._resources: processed_template["Resources"] = self.resolve_attribute(self._resources, ignore_errors) + self.resolve_resources_assets(ignore_errors) if self._outputs: processed_template["Outputs"] = self.resolve_attribute(self._outputs, ignore_errors) return processed_template + def resolve_resources_assets(self, ignore_errors): + for _, resource in self._resources.items(): + if isinstance(resource, Resource): + for asset in resource.assets: + if isinstance(asset, S3Asset): + asset.source_path = self.intrinsic_property_resolver( + asset.source_path, ignore_errors, parent_function=asset.source_property + ) + elif isinstance(asset, ImageAsset): + asset.source_local_image = self.intrinsic_property_resolver( + asset.source_local_image, ignore_errors, parent_function=asset.source_property + ) + def resolve_attribute(self, cloud_formation_property, ignore_errors=False): """ This will parse through every entry in a CloudFormation root key and resolve them based on the symbol_resolver. From a30bb8b1328e34d6ec4605360c41a10886ff4b72 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Wed, 7 Jul 2021 05:27:13 -0700 Subject: [PATCH 22/38] fix two log diff error --- tests/integration/deploy/test_deploy_command.py | 2 +- tests/integration/package/test_package_command_image.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/deploy/test_deploy_command.py b/tests/integration/deploy/test_deploy_command.py index 9653aefd57..30537fd5d9 100644 --- a/tests/integration/deploy/test_deploy_command.py +++ b/tests/integration/deploy/test_deploy_command.py @@ -306,7 +306,7 @@ def test_deploy_without_s3_bucket(self, template_file): self.assertEqual(deploy_process_execute.process.returncode, 1) self.assertIn( bytes( - f"S3 Bucket not specified, use --s3-bucket to specify a bucket name or run sam deploy --guided", + f"Cannot skip both --resolve-s3 and --s3-bucket parameters. Please provide one of these arguments.", encoding="utf-8", ), deploy_process_execute.stderr, diff --git a/tests/integration/package/test_package_command_image.py b/tests/integration/package/test_package_command_image.py index 8534c68e07..6eb2dbd047 100644 --- a/tests/integration/package/test_package_command_image.py +++ b/tests/integration/package/test_package_command_image.py @@ -57,7 +57,7 @@ def test_package_template_without_image_repository(self, template_file): raise process_stderr = stderr.strip() - self.assertIn("Error: Missing option '--image-repository'", process_stderr.decode("utf-8")) + self.assertIn("Error: Cannot skip both --resolve-s3 and --s3-bucket parameters. Please provide one of these arguments.", process_stderr.decode("utf-8")) self.assertEqual(2, process.returncode) @parameterized.expand( From e17642661600f223f53375b9b9ce992be64f78ff Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 7 Jul 2021 08:42:01 -0700 Subject: [PATCH 23/38] Fix recognizing assets in CFN project --- samcli/lib/iac/cfn_iac.py | 6 ++++-- samcli/lib/package/artifact_exporter.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/samcli/lib/iac/cfn_iac.py b/samcli/lib/iac/cfn_iac.py index 39ee87b73d..80be46276d 100644 --- a/samcli/lib/iac/cfn_iac.py +++ b/samcli/lib/iac/cfn_iac.py @@ -13,6 +13,7 @@ RESOURCES_WITH_LOCAL_PATHS, NESTED_STACKS_RESOURCES, ) +from samcli.lib.utils.packagetype import IMAGE, ZIP from samcli.lib.iac.interface import ( IacPlugin, ImageAsset, @@ -71,6 +72,7 @@ def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = resource_id = resource.item_id resource_type = resource.get("Type", None) properties = resource.get("Properties", {}) + package_type = properties.get("PackageType", ZIP) resource_assets = [] @@ -81,7 +83,7 @@ def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = if resource_type in RESOURCES_WITH_LOCAL_PATHS: for path_prop_name in RESOURCES_WITH_LOCAL_PATHS[resource_type]: asset_path = jmespath.search(path_prop_name, properties) - if is_local_path(asset_path): + if is_local_path(asset_path) and package_type == ZIP: reference_path = base_dir if resource_type in BASE_DIR_RESOURCES else os.path.dirname(path) asset_path = get_local_path(asset_path, reference_path) asset = S3Asset(source_path=asset_path, source_property=path_prop_name) @@ -91,7 +93,7 @@ def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = if resource_type in RESOURCES_WITH_IMAGE_COMPONENT: for path_prop_name in RESOURCES_WITH_IMAGE_COMPONENT[resource_type]: asset_path = jmespath.search(path_prop_name, properties) - if asset_path: + if asset_path and package_type == IMAGE: asset = ImageAsset(source_local_image=asset_path, source_property=path_prop_name) resource_assets.append(asset) stack.assets.append(asset) diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index d986ccf261..0a316b1472 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -196,7 +196,7 @@ def _export_metadata(self): if exporter_class.RESOURCE_TYPE != metadata_type: continue - exporter = exporter_class(self.uploaders, self.code_signer) + exporter = exporter_class(self.uploaders, self.code_signer, self.iac) exporter.export(metadata_type, metadata_dict, self.template_dir) def _apply_global_values(self): From ce60d3c3fb34ca4e82ee98699859ebe8fa43d1d0 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 7 Jul 2021 09:06:44 -0700 Subject: [PATCH 24/38] Fix artifact_exporter unit test --- tests/unit/lib/package/test_artifact_exporter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index e02eab9c2e..4cf326a611 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -1154,9 +1154,9 @@ def test_template_export_metadata(self): exported_template = template_exporter.export() self.assertEqual(exported_template, template_dict) - metadata_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) + metadata_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, self.iac_mock) metadata_type1_instance.export.assert_called_once_with("metadata_type1", mock.ANY, template_dir) - metadata_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock) + metadata_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, self.iac_mock) metadata_type2_instance.export.assert_called_once_with("metadata_type2", mock.ANY, template_dir) def test_template_export(self): From 5daaf29e339c7047710457bc55c02bfd922a6cd8 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 7 Jul 2021 12:00:43 -0700 Subject: [PATCH 25/38] Fix handling packageable resources in Metadata --- samcli/lib/iac/cfn_iac.py | 16 ++++++++++++++++ samcli/lib/iac/interface.py | 6 ++++++ samcli/lib/package/artifact_exporter.py | 2 +- .../package/test_package_command_image.py | 5 ++++- tests/unit/lib/package/test_artifact_exporter.py | 4 ++-- 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/samcli/lib/iac/cfn_iac.py b/samcli/lib/iac/cfn_iac.py index 80be46276d..8af89005f5 100644 --- a/samcli/lib/iac/cfn_iac.py +++ b/samcli/lib/iac/cfn_iac.py @@ -9,6 +9,7 @@ import jmespath from samcli.commands._utils.resources import ( + METADATA_WITH_LOCAL_PATHS, RESOURCES_WITH_IMAGE_COMPONENT, RESOURCES_WITH_LOCAL_PATHS, NESTED_STACKS_RESOURCES, @@ -53,6 +54,7 @@ def get_project(self, lookup_paths: List[LookupPath]) -> Project: return Project(stacks) + # pylint: disable=too-many-branches def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = None) -> Stack: assets: List[Asset] = [] @@ -100,6 +102,20 @@ def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = resource.assets = resource_assets + metadata_section = stack.get("Metadata", DictSection()) + for metadata in metadata_section.section_items: + metadata_type = metadata.item_id + metadata_body = metadata.body + metadata_assets = [] + if metadata_type in METADATA_WITH_LOCAL_PATHS: + for path_prop_name in METADATA_WITH_LOCAL_PATHS[metadata_type]: + asset_path = jmespath.search(path_prop_name, metadata_body) + asset = S3Asset(source_path=asset_path, source_property=path_prop_name) + metadata_assets.append(asset) + stack.assets.append(asset) + + metadata.assets = metadata_assets + stack.extra_details[TEMPLATE_PATH_KEY] = path return stack diff --git a/samcli/lib/iac/interface.py b/samcli/lib/iac/interface.py index 5a8c2ea27f..53b5bab6fe 100644 --- a/samcli/lib/iac/interface.py +++ b/samcli/lib/iac/interface.py @@ -373,6 +373,12 @@ def extra_details(self) -> Dict[str, Any]: def extra_details(self, extra_details: Dict[str, Any]) -> None: self._extra_details = extra_details + def is_packageable(self): + """ + return if the resource is packageable + """ + return bool(self.assets) + def __setitem__(self, k: str, v: Any) -> None: self._body[k] = v diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index 0a316b1472..25ac59f6e5 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -197,7 +197,7 @@ def _export_metadata(self): continue exporter = exporter_class(self.uploaders, self.code_signer, self.iac) - exporter.export(metadata_type, metadata_dict, self.template_dir) + exporter.export(metadata_dict, self.template_dir) def _apply_global_values(self): """ diff --git a/tests/integration/package/test_package_command_image.py b/tests/integration/package/test_package_command_image.py index 6eb2dbd047..117a03b253 100644 --- a/tests/integration/package/test_package_command_image.py +++ b/tests/integration/package/test_package_command_image.py @@ -57,7 +57,10 @@ def test_package_template_without_image_repository(self, template_file): raise process_stderr = stderr.strip() - self.assertIn("Error: Cannot skip both --resolve-s3 and --s3-bucket parameters. Please provide one of these arguments.", process_stderr.decode("utf-8")) + self.assertIn( + "Error: Cannot skip both --resolve-s3 and --s3-bucket parameters. Please provide one of these arguments.", + process_stderr.decode("utf-8"), + ) self.assertEqual(2, process.returncode) @parameterized.expand( diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index 4cf326a611..eda6c2324b 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -1155,9 +1155,9 @@ def test_template_export_metadata(self): self.assertEqual(exported_template, template_dict) metadata_type1_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - metadata_type1_instance.export.assert_called_once_with("metadata_type1", mock.ANY, template_dir) + metadata_type1_instance.export.assert_called_once_with(mock.ANY, template_dir) metadata_type2_class.assert_called_once_with(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - metadata_type2_instance.export.assert_called_once_with("metadata_type2", mock.ANY, template_dir) + metadata_type2_instance.export.assert_called_once_with(mock.ANY, template_dir) def test_template_export(self): parent_dir = os.path.sep From 1dafb2ffa182523df27f8ec8d98847e26d515639 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 7 Jul 2021 16:17:56 -0700 Subject: [PATCH 26/38] Fix handling of Metadata resource in artifact exporter --- samcli/lib/package/artifact_exporter.py | 12 +++- samcli/lib/package/packageable_resources.py | 51 +++++++++++++--- .../package/test_package_command_image.py | 5 +- .../lib/package/test_artifact_exporter.py | 60 +++++++++---------- 4 files changed, 83 insertions(+), 45 deletions(-) diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index 25ac59f6e5..37b1384504 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -40,7 +40,7 @@ from samcli.lib.package.utils import is_local_folder, mktempfile, is_s3_url, is_local_file, make_abs_path from samcli.lib.utils.packagetype import ZIP from samcli.yamlhelper import yaml_dump -from samcli.lib.iac.interface import Stack as IacStack, IacPlugin +from samcli.lib.iac.interface import Stack as IacStack, IacPlugin, Resource as IacResource, DictSectionItem # NOTE: sriram-mv, A cyclic dependency on `Template` needs to be broken. @@ -72,7 +72,15 @@ def do_export(self, resource, parent_dir): and set property to URL of the uploaded S3 template """ - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body + + if resource_dict is None: + return + asset = resource.assets[0] template_path = asset.source_path diff --git a/samcli/lib/package/packageable_resources.py b/samcli/lib/package/packageable_resources.py index c25114f6c9..0c8f33ed91 100644 --- a/samcli/lib/package/packageable_resources.py +++ b/samcli/lib/package/packageable_resources.py @@ -9,7 +9,7 @@ from botocore.utils import set_value_from_jmespath from samcli.commands.package import exceptions -from samcli.lib.iac.interface import S3Asset, ImageAsset +from samcli.lib.iac.interface import S3Asset, ImageAsset, DictSectionItem, Resource as IacResource from samcli.lib.package.ecr_uploader import ECRUploader from samcli.lib.package.s3_uploader import S3Uploader from samcli.lib.package.uploaders import Destination, Uploaders @@ -100,7 +100,12 @@ class ResourceZip(Resource): # FIXME: add type annotation once MRO fixed in Iac interface def export(self, resource, parent_dir): resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body + if resource_dict is None: return @@ -146,7 +151,11 @@ def do_export(self, resource, parent_dir): # code signer only accepts files which has '.zip' extension in it # so package artifact with '.zip' if it is required to be signed resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body if not (resource.assets and isinstance(resource.assets[0], S3Asset)): return @@ -166,7 +175,7 @@ def do_export(self, resource, parent_dir): uploaded_url = self.code_signer.sign_package( resource_id, uploaded_url, self.uploader.get_version_of_artifact(uploaded_url) ) - if self.iac.should_update_property_after_package: + if self.iac.should_update_property_after_package(asset): set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) @@ -185,7 +194,12 @@ class ResourceImageDict(Resource): # FIXME: add type annotation once MRO fixed in Iac interface def export(self, resource, parent_dir): resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body + if resource_dict is None: return @@ -213,7 +227,11 @@ def do_export(self, resource, parent_dir): uploaded URL. """ resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body if not (resource.assets and isinstance(resource.assets[0], ImageAsset)): return @@ -240,7 +258,12 @@ class ResourceImage(Resource): # FIXME: add type annotation once MRO fixed in Iac interface def export(self, resource, parent_dir): resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body + if resource_dict is None: return @@ -267,7 +290,12 @@ def do_export(self, resource, parent_dir): URL of the uploaded object """ resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body + uploaded_url = upload_local_image_artifacts( resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader ) @@ -295,7 +323,12 @@ def do_export(self, resource, parent_dir): of the uploaded object """ resource_id = resource.key - resource_dict = resource.get("Properties", {}) + resource_dict = None + if isinstance(resource, IacResource): + resource_dict = resource.get("Properties") + elif isinstance(resource, DictSectionItem): + resource_dict = resource.body + asset = resource.assets[0] artifact_s3_url = upload_local_artifacts(resource_id, asset, self.PROPERTY_NAME, parent_dir, self.uploader) diff --git a/tests/integration/package/test_package_command_image.py b/tests/integration/package/test_package_command_image.py index 117a03b253..8534c68e07 100644 --- a/tests/integration/package/test_package_command_image.py +++ b/tests/integration/package/test_package_command_image.py @@ -57,10 +57,7 @@ def test_package_template_without_image_repository(self, template_file): raise process_stderr = stderr.strip() - self.assertIn( - "Error: Cannot skip both --resolve-s3 and --s3-bucket parameters. Please provide one of these arguments.", - process_stderr.decode("utf-8"), - ) + self.assertIn("Error: Missing option '--image-repository'", process_stderr.decode("utf-8")) self.assertEqual(2, process.returncode) @parameterized.expand( diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index eda6c2324b..f459ca5d40 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -52,7 +52,7 @@ ResourceZip, ResourceImage, ) -from samcli.lib.iac.interface import Stack as IacStack, S3Asset +from samcli.lib.iac.interface import Stack as IacStack, S3Asset, Resource as IacResource class TestArtifactExporter(unittest.TestCase): @@ -71,7 +71,7 @@ def get_mock(destination: Destination): self.code_signer_mock.should_sign_package.return_value = False self.iac_mock = Mock() - self.iac_mock.should_update_property_after_package = True + self.iac_mock.should_update_property_after_package.return_value = True self.iac_mock.update_resource_after_packaging = Mock() def test_all_resources_export(self): @@ -118,7 +118,7 @@ def test_invalid_export_resource(self): resource_obj = ServerlessFunctionResource( uploaders=self.uploaders_mock, code_signer=code_signer_mock, iac=self.iac_mock ) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.get.return_value = {"InlineCode": "code"} iac_resource_mock.is_packageable.return_value = False @@ -141,7 +141,7 @@ def _helper_verify_export_resources( uploaders_mock = Mock() uploaders_mock.get = Mock(return_value=s3_uploader_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] @@ -427,7 +427,7 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] asset_mock = iac_resource_mock.assets[0] @@ -458,7 +458,7 @@ class MockResource(ResourceImage): resource = MockResource(self.uploaders_mock, None, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -489,7 +489,7 @@ class MockResource(ResourceImage): resource = MockResource(self.uploaders_mock, None, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -517,7 +517,7 @@ class MockResource(ResourceImage): resource = MockResource(self.uploaders_mock, None, self.iac_mock) original_image = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -541,7 +541,7 @@ class MockResource(ResourceImage): resource = MockResource(self.uploaders_mock, None, self.iac_mock) original_image = None - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -572,7 +572,7 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) original_path = "/path/to/file" - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] asset_mock = iac_resource_mock.assets[0] @@ -628,7 +628,7 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) original_path = "/path/to/zip_file" - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] asset_mock = iac_resource_mock.assets[0] @@ -678,7 +678,7 @@ class MockResourceNoForceZip(ResourceZip): resource = MockResourceNoForceZip(self.uploaders_mock, self.code_signer_mock, self.iac_mock) original_path = "/path/to/file" - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] asset_mock = iac_resource_mock.assets[0] @@ -716,7 +716,7 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] asset_mock = iac_resource_mock.assets[0] @@ -745,7 +745,7 @@ class MockResource(ResourceZip): PACKAGE_NULL_PROPERTY = False resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -769,7 +769,7 @@ class MockResource(ResourceZip): PROPERTY_NAME = "foo" resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.assets = [MagicMock(spec=S3Asset)] iac_resource_mock.key = "id" parent_dir = "dir" @@ -800,7 +800,7 @@ class MockResource(ResourceWithS3UrlDict): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) # Case 1: Property value is a path to file - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -837,7 +837,7 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = [MagicMock(spec=S3Asset)] asset_mock = iac_resource_mock.assets[0] @@ -853,7 +853,7 @@ class MockResource(ResourceZip): def test_export_cloudformation_stack(self, TemplateMock): stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME exported_template_dict = {"foo": "bar"} @@ -896,7 +896,7 @@ def test_export_cloudformation_stack_no_nested_stack(self): do_export_mock = Mock() stack_resource.do_export = do_export_mock - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.nested_stack = None stack_resource.export(iac_resource_mock, "dir") @@ -904,7 +904,7 @@ def test_export_cloudformation_stack_no_nested_stack(self): def test_export_cloudformation_stack_no_upload_path_is_s3url(self): stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" @@ -920,7 +920,7 @@ def test_export_cloudformation_stack_no_upload_path_is_s3url(self): def test_export_cloudformation_stack_no_upload_path_is_httpsurl(self): stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "https://s3.amazonaws.com/hello/world" @@ -937,7 +937,7 @@ def test_export_cloudformation_stack_no_upload_path_is_httpsurl(self): def test_export_cloudformation_stack_no_upload_path_is_s3_region_httpsurl(self): stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "https://s3.some-valid-region.amazonaws.com/hello/world" @@ -952,7 +952,7 @@ def test_export_cloudformation_stack_no_upload_path_is_s3_region_httpsurl(self): def test_export_cloudformation_stack_no_upload_path_is_empty(self): stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" @@ -969,7 +969,7 @@ def test_export_cloudformation_stack_no_upload_path_is_empty(self): def test_export_cloudformation_stack_no_upload_path_not_file(self): stack_resource = CloudFormationStackResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" @@ -993,7 +993,7 @@ def test_export_cloudformation_stack_no_upload_path_not_file(self): def test_export_serverless_application(self, TemplateMock): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME iac_resource_mock.assets = MagicMock() @@ -1033,7 +1033,7 @@ def test_export_serverless_application(self, TemplateMock): def test_export_serverless_application_no_upload_path_is_s3url(self): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "s3://hello/world" @@ -1049,7 +1049,7 @@ def test_export_serverless_application_no_upload_path_is_s3url(self): def test_export_serverless_application_no_upload_path_is_httpsurl(self): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME s3_url = "https://s3.amazonaws.com/hello/world" @@ -1065,7 +1065,7 @@ def test_export_serverless_application_no_upload_path_is_httpsurl(self): def test_export_serverless_application_no_upload_path_is_empty(self): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME @@ -1080,7 +1080,7 @@ def test_export_serverless_application_no_upload_path_is_empty(self): def test_export_serverless_application_no_upload_path_not_file(self): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" iac_resource_mock.assets = MagicMock() iac_resource_mock.assets[0] = Mock() @@ -1101,7 +1101,7 @@ def test_export_serverless_application_no_upload_path_not_file(self): def test_export_serverless_application_no_upload_path_is_dictionary(self): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) - iac_resource_mock = MagicMock() + iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME From a192366247437c1d4a00064a28f4dd5a129d8b84 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 7 Jul 2021 17:56:03 -0700 Subject: [PATCH 27/38] Fix integ test - test_deploy_without_stack_name --- tests/integration/deploy/test_deploy_command.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/deploy/test_deploy_command.py b/tests/integration/deploy/test_deploy_command.py index 30537fd5d9..d942d09ead 100644 --- a/tests/integration/deploy/test_deploy_command.py +++ b/tests/integration/deploy/test_deploy_command.py @@ -321,6 +321,7 @@ def test_deploy_without_stack_name(self, template_file): template_file=template_path, capabilities="CAPABILITY_IAM", s3_prefix="integ_deploy", + s3_bucket=self.s3_bucket.name, force_upload=True, notification_arns=self.sns_arn, parameter_overrides="Parameter=Clarity", From 9bb6bb7d2f71552423c0667866a0f4729ec5245e Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Wed, 7 Jul 2021 21:44:43 -0700 Subject: [PATCH 28/38] Handling missing stack_name in iac_validator --- samcli/commands/_utils/iac_validations.py | 9 +++++++++ .../commands/_utils/test_iac_validations.py | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/samcli/commands/_utils/iac_validations.py b/samcli/commands/_utils/iac_validations.py index 3008ecfc6c..fdfd158993 100644 --- a/samcli/commands/_utils/iac_validations.py +++ b/samcli/commands/_utils/iac_validations.py @@ -68,6 +68,15 @@ def wrapped(*args, **kwargs): message="More than one stack found. Use '--stack-name' to specify the stack.", ) + command = ctx.command.name + if command == "deploy" and not stack_name and not guided: + raise click.BadOptionUsage( + option_name="--stack-name", + ctx=ctx, + message="Missing option '--stack-name', 'sam deploy --guided' can " + "be used to provide and save needed parameters for future deploys.", + ) + return func(*args, **kwargs) return wrapped diff --git a/tests/unit/commands/_utils/test_iac_validations.py b/tests/unit/commands/_utils/test_iac_validations.py index 39f09e1102..562c0aa35b 100644 --- a/tests/unit/commands/_utils/test_iac_validations.py +++ b/tests/unit/commands/_utils/test_iac_validations.py @@ -46,6 +46,25 @@ def test_validation_success_cfn_require_stack(self, click_mock): project_mock = Mock() self.func_require_stack(project=project_mock) + @patch("samcli.commands._utils.iac_validations.click") + def test_validation_fail_cfn_missing_stack_name_when_deploy(self, click_mock): + params = {"project_type": "CFN"} + context_mock = MagicMock() + context_mock.params.get.side_effect = _make_ctx_params_side_effect_func(params) + context_mock.command.name = "deploy" + click_mock.get_current_context.return_value = context_mock + click_mock.BadOptionUsage = click.BadOptionUsage + + project_mock = Mock() + with self.assertRaises(click_mock.BadOptionUsage) as ex: + self.func_require_stack(project=project_mock) + self.assertEqual(ex.exception.option_name, "--stack-name") + self.assertEqual( + ex.exception.message, + "Missing option '--stack-name', 'sam deploy --guided' can " + "be used to provide and save needed parameters for future deploys.", + ) + @patch("samcli.commands._utils.iac_validations.click") def test_validation_fail_cfn_invalid_options(self, click_mock): params = { From 503a27f0504e9c7f13e0233b790001916b9b30f0 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Thu, 8 Jul 2021 09:55:38 -0700 Subject: [PATCH 29/38] Add more tests --- samcli/lib/iac/cdk/exceptions.py | 15 --- samcli/lib/iac/cdk/helpers.py | 14 +- samcli/lib/package/packageable_resources.py | 9 +- tests/unit/lib/iac/test_helpers.py | 67 ++++++++++ .../lib/package/test_artifact_exporter.py | 125 ++++++++++++++++-- 5 files changed, 186 insertions(+), 44 deletions(-) create mode 100644 tests/unit/lib/iac/test_helpers.py diff --git a/samcli/lib/iac/cdk/exceptions.py b/samcli/lib/iac/cdk/exceptions.py index 231f200fe1..435aee72d1 100644 --- a/samcli/lib/iac/cdk/exceptions.py +++ b/samcli/lib/iac/cdk/exceptions.py @@ -6,21 +6,6 @@ from samcli.commands.exceptions import UserException -class UnsupportedCloudAssemblySchemaVersionError(Exception): - def __init__(self, cloud_assembly_schema_version: str): - msg = ( - "Your cloud assembly schema version '{cloud_assembly_schema_version}' is not supported, " - "probably because you are running an old version of CDK. Please upgrade your CDK." - ) - Exception.__init__(self, msg.format(cloud_assembly_schema_version=cloud_assembly_schema_version)) - - -class UnsupportedCdkFeatureError(Exception): - def __init__(self, reason: str): - msg = "You are using a CDK feature that is currently not supported by SAM CLI yet. Reason: '{reason}'" - Exception.__init__(self, msg.format(reason=reason)) - - class InvalidCloudAssemblyError(Exception): def __init__(self, missing_files: Optional[list] = None): if missing_files is None: diff --git a/samcli/lib/iac/cdk/helpers.py b/samcli/lib/iac/cdk/helpers.py index 27f09fa8a6..e07a0f0517 100644 --- a/samcli/lib/iac/cdk/helpers.py +++ b/samcli/lib/iac/cdk/helpers.py @@ -9,21 +9,9 @@ Pattern, ) from collections.abc import Mapping -from samcli.lib.iac.cdk.constants import CDK_PATH_DELIMITER - -LOG = logging.getLogger(__name__) -CDK_PATH_DELIMITER = "/" -def nested_stack_resource_path_to_short_path(nested_stack_path: str) -> str: - """ - return short path for given nested stack resource path - Example: - Root/NS1/NS2.NestedStack/NS2.NestedStackResource -> Root/NS1/NS2 - """ - needed_path_parts = nested_stack_path.split(CDK_PATH_DELIMITER)[:-1] - needed_path_parts[-1] = needed_path_parts[-1].rsplit(".NestedStack", 1)[0] - return CDK_PATH_DELIMITER.join(needed_path_parts) +LOG = logging.getLogger(__name__) def get_nested_stack_asset_id(nested_stack_resource: Dict) -> Optional[str]: diff --git a/samcli/lib/package/packageable_resources.py b/samcli/lib/package/packageable_resources.py index 0c8f33ed91..dc6f6b754e 100644 --- a/samcli/lib/package/packageable_resources.py +++ b/samcli/lib/package/packageable_resources.py @@ -240,7 +240,7 @@ def do_export(self, resource, parent_dir): resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader ) self.iac.update_resource_after_packaging(resource) - if self.iac.should_update_property_after_package: + if self.iac.should_update_property_after_package(resource.assets[0]): set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, {self.EXPORT_PROPERTY_CODE_KEY: uploaded_url}) @@ -296,11 +296,14 @@ def do_export(self, resource, parent_dir): elif isinstance(resource, DictSectionItem): resource_dict = resource.body + if not (resource.assets and isinstance(resource.assets[0], ImageAsset)): + return + uploaded_url = upload_local_image_artifacts( resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader ) self.iac.update_resource_after_packaging(resource) - if self.iac.should_update_property_after_package: + if self.iac.should_update_property_after_package(resource.assets[0]): set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) @@ -343,7 +346,7 @@ def do_export(self, resource, parent_dir): asset.object_key = parsed_url[self.OBJECT_KEY_PROPERTY] asset.object_version = parsed_url.get(self.VERSION_PROPERTY, None) self.iac.update_resource_after_packaging(resource) - if self.iac.should_update_property_after_package: + if self.iac.should_update_property_after_package(asset): set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, parsed_url) diff --git a/tests/unit/lib/iac/test_helpers.py b/tests/unit/lib/iac/test_helpers.py new file mode 100644 index 0000000000..622bca5374 --- /dev/null +++ b/tests/unit/lib/iac/test_helpers.py @@ -0,0 +1,67 @@ +from unittest import TestCase +from unittest.mock import Mock, MagicMock, patch + +from samcli.lib.iac.interface import LookupPathType, ProjectTypes, LookupPath +from samcli.lib.iac.utils.helpers import get_iac_plugin, inject_iac_plugin + + +class TestGetIacPlugin(TestCase): + @patch("samcli.lib.iac.utils.helpers.CfnIacPlugin") + def test_get_iac_plugin_cfn_with_build(self, CfnIacPluginMock): + cfn_iac_plugin_mock = Mock() + cfn_iac_plugin_mock.get_project = Mock() + cfn_iac_plugin_mock.get_project.return_value = Mock() + CfnIacPluginMock.return_value = cfn_iac_plugin_mock + command_params = MagicMock() + command_params.get.return_value = "some_build_dir" + + iac_plugin, project = get_iac_plugin(ProjectTypes.CFN.value, command_params, True) + CfnIacPluginMock.assert_called_once_with(command_params) + self.assertEqual(iac_plugin, cfn_iac_plugin_mock) + self.assertEqual(project, cfn_iac_plugin_mock.get_project.return_value) + + @patch("samcli.lib.iac.utils.helpers.CdkPlugin") + def test_get_iac_plugin_cdk_with_build(self, CdkIacPluginMock): + cdk_iac_plugin_mock = Mock() + cdk_iac_plugin_mock.get_project = Mock() + cdk_iac_plugin_mock.get_project.return_value = Mock() + CdkIacPluginMock.return_value = cdk_iac_plugin_mock + command_params = MagicMock() + command_params.get.return_value = "some_build_dir" + + iac_plugin, project = get_iac_plugin(ProjectTypes.CDK.value, command_params, True) + CdkIacPluginMock.assert_called_once_with(command_params) + self.assertEqual(iac_plugin, cdk_iac_plugin_mock) + self.assertEqual(project, cdk_iac_plugin_mock.get_project.return_value) + + +class TestInjectIacPlugin(TestCase): + @patch("samcli.lib.iac.utils.helpers.get_iac_plugin") + def test_inject_iac_plugin_cfn_with_build(self, get_iac_plugin_mock): + iac_plugin_mock = Mock() + project_mock = Mock() + get_iac_plugin_mock.return_value = (iac_plugin_mock, project_mock) + + @inject_iac_plugin(True) + def func(*args, **kwargs): + return kwargs + + result = func(project_type=ProjectTypes.CFN.value) + + get_iac_plugin_mock.assert_called_once_with( + ProjectTypes.CFN.value, + { + "project_type": ProjectTypes.CFN.value, + "iac": iac_plugin_mock, + "project": project_mock, + }, + True, + ) + self.assertEqual( + result, + { + "project_type": ProjectTypes.CFN.value, + "iac": iac_plugin_mock, + "project": project_mock, + }, + ) diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index f459ca5d40..98c47959a3 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -51,8 +51,9 @@ CloudFormationResourceVersionSchemaHandlerPackage, ResourceZip, ResourceImage, + ResourceImageDict, ) -from samcli.lib.iac.interface import Stack as IacStack, S3Asset, Resource as IacResource +from samcli.lib.iac.interface import ImageAsset, Stack as IacStack, S3Asset, Resource as IacResource class TestArtifactExporter(unittest.TestCase): @@ -460,11 +461,10 @@ class MockResource(ResourceImage): iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = "image:latest" asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.assets = [asset_mock] resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "image:latest" iac_resource_mock.get.return_value = resource_dict @@ -491,11 +491,10 @@ class MockResource(ResourceImage): iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = "image:latest" asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.assets = [asset_mock] resource_dict = {} original_image = "image:latest" resource_dict[resource.PROPERTY_NAME] = original_image @@ -519,10 +518,9 @@ class MockResource(ResourceImage): original_image = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = original_image + iac_resource_mock.assets = [asset_mock] resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image iac_resource_mock.get.return_value = resource_dict @@ -543,10 +541,111 @@ class MockResource(ResourceImage): original_image = None iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=ImageAsset) + asset_mock.source_local_image = original_image + iac_resource_mock.assets = [asset_mock] + resource_dict = {} + resource_dict[resource.PROPERTY_NAME] = original_image + iac_resource_mock.get.return_value = resource_dict + parent_dir = "dir" + + with self.assertRaises(ExportFailedError): + resource.export(iac_resource_mock, parent_dir) + + @patch("samcli.lib.package.packageable_resources.upload_local_image_artifacts") + def test_resource_image_dict(self, upload_local_image_artifacts_mock): + # Property value is a path to an image + + class MockResource(ResourceImageDict): + PROPERTY_NAME = "foo" + + resource = MockResource(self.uploaders_mock, None, self.iac_mock) + + iac_resource_mock = MagicMock(spec=IacResource) + iac_resource_mock.key = "id" + asset_mock = Mock(spec=ImageAsset) + asset_mock.source_local_image = "image:latest" + asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.assets = [asset_mock] + resource_dict = {} + resource_dict[resource.PROPERTY_NAME] = "image:latest" + iac_resource_mock.get.return_value = resource_dict + parent_dir = "dir" + ecr_url = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" + + upload_local_image_artifacts_mock.return_value = ecr_url + + resource.export(iac_resource_mock, parent_dir) + + upload_local_image_artifacts_mock.assert_called_once_with( + iac_resource_mock.key, asset_mock, resource.PROPERTY_NAME, parent_dir, self.ecr_uploader_mock + ) + + self.assertEqual(resource_dict[resource.PROPERTY_NAME], {"ImageUri": ecr_url}) + + def test_resource_image_dict_package_success(self): + # Property value is set to an image + + class MockResource(ResourceImageDict): + PROPERTY_NAME = "foo" + + resource = MockResource(self.uploaders_mock, None, self.iac_mock) + + iac_resource_mock = MagicMock(spec=IacResource) + iac_resource_mock.key = "id" + asset_mock = Mock(spec=ImageAsset) + asset_mock.source_local_image = "image:latest" + asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.assets = [asset_mock] + resource_dict = {} + original_image = "image:latest" + resource_dict[resource.PROPERTY_NAME] = original_image + iac_resource_mock.get.return_value = resource_dict + parent_dir = "dir" + ecr_url = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" + self.ecr_uploader_mock.upload.return_value = ecr_url + + resource.export(iac_resource_mock, parent_dir) + + self.assertEqual(resource_dict[resource.PROPERTY_NAME], {"ImageUri": ecr_url}) + + def test_resource_image_dict_non_package_image_already_remote(self): + # Property value is set to an ecr image + + class MockResource(ResourceImageDict): + PROPERTY_NAME = "foo" + + resource = MockResource(self.uploaders_mock, None, self.iac_mock) + + original_image = "123456789.dkr.ecr.us-east-1.amazonaws.com/sam-cli" + iac_resource_mock = MagicMock(spec=IacResource) + iac_resource_mock.key = "id" + asset_mock = Mock(spec=ImageAsset) + asset_mock.source_local_image = original_image + iac_resource_mock.assets = [asset_mock] + resource_dict = {} + resource_dict[resource.PROPERTY_NAME] = original_image + iac_resource_mock.get.return_value = resource_dict + parent_dir = "dir" + + resource.export(iac_resource_mock, parent_dir) + + self.assertEqual(resource_dict[resource.PROPERTY_NAME], {"ImageUri": original_image}) + + def test_resource_image_dict_no_image_present(self): + # Property value is set to an ecr image + + class MockResource(ResourceImageDict): + PROPERTY_NAME = "foo" + + resource = MockResource(self.uploaders_mock, None, self.iac_mock) + + original_image = None + iac_resource_mock = MagicMock(spec=IacResource) + iac_resource_mock.key = "id" + asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = original_image + iac_resource_mock.assets = [asset_mock] resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image iac_resource_mock.get.return_value = resource_dict From 1ee28ded9717f6a2a59e12c25b4ebe82c9628120 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Sat, 10 Jul 2021 20:57:51 -0700 Subject: [PATCH 30/38] Improve package regression log --- tests/regression/package/regression_package_base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/regression/package/regression_package_base.py b/tests/regression/package/regression_package_base.py index 4593a9ef59..776b982d12 100644 --- a/tests/regression/package/regression_package_base.py +++ b/tests/regression/package/regression_package_base.py @@ -112,4 +112,9 @@ def regression_check(self, args): output_sam = json.loads(output_sam) output_aws = json.loads(output_aws) + if output_aws != output_sam: + print("output_aws and output_sam are different!!!") + print(f"output_aws = {output_aws}") + print(f"output_sam = {output_sam}") + self.assertEqual(output_sam, output_aws) From de9588186e74499ec8acb6adfd0873a8a13608ca Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Sat, 10 Jul 2021 21:07:49 -0700 Subject: [PATCH 31/38] Increase rerun number on two flaky tests test_all_containers_are_initialized_before_any_invoke/test_no_new_created_containers_after_lambda_function_invoke --- tests/integration/local/start_api/test_start_api.py | 4 ++-- tests/integration/local/start_lambda/test_start_lambda.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/local/start_api/test_start_api.py b/tests/integration/local/start_api/test_start_api.py index e7e5ad59a1..8aa146b254 100644 --- a/tests/integration/local/start_api/test_start_api.py +++ b/tests/integration/local/start_api/test_start_api.py @@ -1709,7 +1709,7 @@ class TestWarmContainersInitialization(TestWarmContainersBaseClass): mode_env_variable = str(uuid.uuid4()) parameter_overrides = {"ModeEnvVariable": mode_env_variable} - @pytest.mark.flaky(reruns=3) + @pytest.mark.flaky(reruns=5) @pytest.mark.timeout(timeout=600, method="thread") def test_all_containers_are_initialized_before_any_invoke(self): initiated_containers = self.count_running_containers() @@ -1723,7 +1723,7 @@ class TestWarmContainersMultipleInvoke(TestWarmContainersBaseClass): mode_env_variable = str(uuid.uuid4()) parameter_overrides = {"ModeEnvVariable": mode_env_variable} - @pytest.mark.flaky(reruns=3) + @pytest.mark.flaky(reruns=5) @pytest.mark.timeout(timeout=600, method="thread") def test_no_new_created_containers_after_lambda_function_invoke(self): initiated_containers_before_invoking_any_function = self.count_running_containers() diff --git a/tests/integration/local/start_lambda/test_start_lambda.py b/tests/integration/local/start_lambda/test_start_lambda.py index ca4a00b5cf..a35eceb1a1 100644 --- a/tests/integration/local/start_lambda/test_start_lambda.py +++ b/tests/integration/local/start_lambda/test_start_lambda.py @@ -271,7 +271,7 @@ class TestWarmContainersInitialization(TestWarmContainersBaseClass): mode_env_variable = str(uuid.uuid4()) parameter_overrides = {"ModeEnvVariable": mode_env_variable} - @pytest.mark.flaky(reruns=3) + @pytest.mark.flaky(reruns=5) @pytest.mark.timeout(timeout=600, method="thread") def test_all_containers_are_initialized_before_any_invoke(self): initiated_containers = self.count_running_containers() @@ -285,7 +285,7 @@ class TestWarmContainersMultipleInvoke(TestWarmContainersBaseClass): mode_env_variable = str(uuid.uuid4()) parameter_overrides = {"ModeEnvVariable": mode_env_variable} - @pytest.mark.flaky(reruns=3) + @pytest.mark.flaky(reruns=5) @pytest.mark.timeout(timeout=600, method="thread") def test_no_new_created_containers_after_lambda_function_invoke(self): From edb43b756b81b2c7d821082c00cf9ae599963c22 Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Sun, 11 Jul 2021 20:19:17 -0700 Subject: [PATCH 32/38] Fix handling of multiple assets in one resource --- samcli/lib/iac/interface.py | 8 ++ samcli/lib/package/artifact_exporter.py | 6 +- samcli/lib/package/packageable_resources.py | 43 +++++---- .../lib/package/test_artifact_exporter.py | 94 +++++++++---------- 4 files changed, 83 insertions(+), 68 deletions(-) diff --git a/samcli/lib/iac/interface.py b/samcli/lib/iac/interface.py index 53b5bab6fe..9df0222711 100644 --- a/samcli/lib/iac/interface.py +++ b/samcli/lib/iac/interface.py @@ -379,6 +379,14 @@ def is_packageable(self): """ return bool(self.assets) + def find_asset_by_source_property(self, source_property: str) -> Optional[Asset]: + if not self.assets: + return None + for asset in self.assets: + if asset.source_property == source_property: + return asset + return None + def __setitem__(self, k: str, v: Any) -> None: self._body[k] = v diff --git a/samcli/lib/package/artifact_exporter.py b/samcli/lib/package/artifact_exporter.py index 37b1384504..4fc268af49 100644 --- a/samcli/lib/package/artifact_exporter.py +++ b/samcli/lib/package/artifact_exporter.py @@ -40,7 +40,7 @@ from samcli.lib.package.utils import is_local_folder, mktempfile, is_s3_url, is_local_file, make_abs_path from samcli.lib.utils.packagetype import ZIP from samcli.yamlhelper import yaml_dump -from samcli.lib.iac.interface import Stack as IacStack, IacPlugin, Resource as IacResource, DictSectionItem +from samcli.lib.iac.interface import Stack as IacStack, IacPlugin, Resource as IacResource, DictSectionItem, S3Asset # NOTE: sriram-mv, A cyclic dependency on `Template` needs to be broken. @@ -81,7 +81,9 @@ def do_export(self, resource, parent_dir): if resource_dict is None: return - asset = resource.assets[0] + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, S3Asset)): + return template_path = asset.source_path if ( diff --git a/samcli/lib/package/packageable_resources.py b/samcli/lib/package/packageable_resources.py index dc6f6b754e..5923418e60 100644 --- a/samcli/lib/package/packageable_resources.py +++ b/samcli/lib/package/packageable_resources.py @@ -115,8 +115,11 @@ def export(self, resource, parent_dir): if not resource.assets: return + # resource.assets contains at lease one asset - asset = resource.assets[0] + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not asset: + return if not asset.source_path and not self.PACKAGE_NULL_PROPERTY: return @@ -157,10 +160,10 @@ def do_export(self, resource, parent_dir): elif isinstance(resource, DictSectionItem): resource_dict = resource.body - if not (resource.assets and isinstance(resource.assets[0], S3Asset)): + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, S3Asset)): return - asset = resource.assets[0] should_sign_package = self.code_signer.should_sign_package(resource_id) artifact_extension = "zip" if should_sign_package else None uploaded_url = upload_local_artifacts( @@ -207,6 +210,10 @@ def export(self, resource, parent_dir): if not resource.is_packageable(): return + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, ImageAsset)): + return + try: self.do_export(resource, parent_dir) @@ -215,7 +222,7 @@ def export(self, resource, parent_dir): raise exceptions.ExportFailedError( resource_id=resource_id, property_name=self.PROPERTY_NAME, - property_value=resource.assets[0].source_local_image, + property_value=asset.source_local_image, ex=ex, ) @@ -233,14 +240,13 @@ def do_export(self, resource, parent_dir): elif isinstance(resource, DictSectionItem): resource_dict = resource.body - if not (resource.assets and isinstance(resource.assets[0], ImageAsset)): + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, ImageAsset)): return - uploaded_url = upload_local_image_artifacts( - resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader - ) + uploaded_url = upload_local_image_artifacts(resource_id, asset, self.PROPERTY_NAME, parent_dir, self.uploader) self.iac.update_resource_after_packaging(resource) - if self.iac.should_update_property_after_package(resource.assets[0]): + if self.iac.should_update_property_after_package(asset): set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, {self.EXPORT_PROPERTY_CODE_KEY: uploaded_url}) @@ -271,6 +277,10 @@ def export(self, resource, parent_dir): if not resource.is_packageable(): return + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, ImageAsset)): + return + try: self.do_export(resource, parent_dir) @@ -279,7 +289,7 @@ def export(self, resource, parent_dir): raise exceptions.ExportFailedError( resource_id=resource_id, property_name=self.PROPERTY_NAME, - property_value=resource.assets[0].source_local_image, + property_value=asset.source_local_image, ex=ex, ) @@ -296,14 +306,13 @@ def do_export(self, resource, parent_dir): elif isinstance(resource, DictSectionItem): resource_dict = resource.body - if not (resource.assets and isinstance(resource.assets[0], ImageAsset)): + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, ImageAsset)): return - uploaded_url = upload_local_image_artifacts( - resource_id, resource.assets[0], self.PROPERTY_NAME, parent_dir, self.uploader - ) + uploaded_url = upload_local_image_artifacts(resource_id, asset, self.PROPERTY_NAME, parent_dir, self.uploader) self.iac.update_resource_after_packaging(resource) - if self.iac.should_update_property_after_package(resource.assets[0]): + if self.iac.should_update_property_after_package(asset): set_value_from_jmespath(resource_dict, self.PROPERTY_NAME, uploaded_url) @@ -332,7 +341,9 @@ def do_export(self, resource, parent_dir): elif isinstance(resource, DictSectionItem): resource_dict = resource.body - asset = resource.assets[0] + asset = resource.find_asset_by_source_property(self.PROPERTY_NAME) + if not (asset is not None and isinstance(asset, S3Asset)): + return artifact_s3_url = upload_local_artifacts(resource_id, asset, self.PROPERTY_NAME, parent_dir, self.uploader) diff --git a/tests/unit/lib/package/test_artifact_exporter.py b/tests/unit/lib/package/test_artifact_exporter.py index 98c47959a3..a978fb6db9 100644 --- a/tests/unit/lib/package/test_artifact_exporter.py +++ b/tests/unit/lib/package/test_artifact_exporter.py @@ -123,7 +123,7 @@ def test_invalid_export_resource(self): iac_resource_mock.key = "id" iac_resource_mock.get.return_value = {"InlineCode": "code"} iac_resource_mock.is_packageable.return_value = False - iac_resource_mock.assets = [] + iac_resource_mock.find_asset_by_source_property.return_value = None parent_dir = "dir" resource_obj.export(iac_resource_mock, parent_dir) upload_local_artifacts_mock.assert_not_called() @@ -144,7 +144,7 @@ def _helper_verify_export_resources( iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + iac_resource_mock.find_asset_by_source_property.return_value = MagicMock(spec=S3Asset) if "." in test_class.PROPERTY_NAME: reversed_property_names = test_class.PROPERTY_NAME.split(".") @@ -171,7 +171,7 @@ def _helper_verify_export_resources( ): upload_local_artifacts_mock.assert_called_once_with( iac_resource_mock.key, - iac_resource_mock.assets[0], + iac_resource_mock.find_asset_by_source_property(), test_class.PROPERTY_NAME, parent_dir, s3_uploader_mock, @@ -179,7 +179,7 @@ def _helper_verify_export_resources( else: upload_local_artifacts_mock.assert_called_once_with( iac_resource_mock.key, - iac_resource_mock.assets[0], + iac_resource_mock.find_asset_by_source_property(), test_class.PROPERTY_NAME, parent_dir, s3_uploader_mock, @@ -288,7 +288,7 @@ def test_upload_local_artifacts_local_file(self, zip_and_upload_mock): artifact_path = handle.name parent_dir = tempfile.gettempdir() - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = artifact_path result = upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) self.assertEqual(result, expected_s3_url) @@ -313,7 +313,7 @@ def test_upload_local_artifacts_local_file_abs_path(self, zip_and_upload_mock): with tempfile.NamedTemporaryFile() as handle: parent_dir = tempfile.gettempdir() artifact_path = make_abs_path(parent_dir, handle.name) - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = artifact_path asset_mock.source_property = property_name @@ -335,7 +335,7 @@ def test_upload_local_artifacts_local_folder(self, zip_and_upload_mock): with self.make_temp_dir() as artifact_path: # Artifact is a file in the temporary directory parent_dir = tempfile.gettempdir() - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = artifact_path asset_mock.source_property = property_name @@ -357,7 +357,7 @@ def test_upload_local_artifacts_no_path(self, zip_and_upload_mock): # If you don't specify a path, we will default to Current Working Dir resource_dict = {} parent_dir = tempfile.gettempdir() - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = None asset_mock.source_property = property_name @@ -374,7 +374,7 @@ def test_upload_local_artifacts_s3_url(self, zip_and_upload_mock): object_s3_url = "s3://foo/bar?versionId=baz" # If URL is already S3 URL, this will be returned without zip/upload - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = object_s3_url asset_mock.source_property = property_name parent_dir = tempfile.gettempdir() @@ -393,14 +393,14 @@ def test_upload_local_artifacts_invalid_value(self, zip_and_upload_mock): with self.assertRaises(exceptions.InvalidLocalPathError): non_existent_file = "some_random_filename" - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = non_existent_file asset_mock.source_property = property_name upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) with self.assertRaises(exceptions.InvalidLocalPathError): non_existent_file = ["invalid datatype"] - asset_mock = Mock() + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = non_existent_file asset_mock.source_property = property_name upload_local_artifacts(resource_id, asset_mock, property_name, parent_dir, self.s3_uploader_mock) @@ -430,10 +430,10 @@ class MockResource(ResourceZip): iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] - asset_mock = iac_resource_mock.assets[0] + asset_mock = MagicMock(spec=S3Asset) asset_mock.source_path = "/path/to/file" asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "/path/to/file" iac_resource_mock.get.return_value = resource_dict @@ -464,7 +464,7 @@ class MockResource(ResourceImage): asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = "image:latest" asset_mock.source_property = resource.PROPERTY_NAME - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "image:latest" iac_resource_mock.get.return_value = resource_dict @@ -494,7 +494,7 @@ class MockResource(ResourceImage): asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = "image:latest" asset_mock.source_property = resource.PROPERTY_NAME - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} original_image = "image:latest" resource_dict[resource.PROPERTY_NAME] = original_image @@ -520,7 +520,7 @@ class MockResource(ResourceImage): iac_resource_mock.key = "id" asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = original_image - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image iac_resource_mock.get.return_value = resource_dict @@ -543,7 +543,7 @@ class MockResource(ResourceImage): iac_resource_mock.key = "id" asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = original_image - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image iac_resource_mock.get.return_value = resource_dict @@ -566,7 +566,7 @@ class MockResource(ResourceImageDict): asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = "image:latest" asset_mock.source_property = resource.PROPERTY_NAME - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "image:latest" iac_resource_mock.get.return_value = resource_dict @@ -596,7 +596,7 @@ class MockResource(ResourceImageDict): asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = "image:latest" asset_mock.source_property = resource.PROPERTY_NAME - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} original_image = "image:latest" resource_dict[resource.PROPERTY_NAME] = original_image @@ -622,7 +622,7 @@ class MockResource(ResourceImageDict): iac_resource_mock.key = "id" asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = original_image - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image iac_resource_mock.get.return_value = resource_dict @@ -645,7 +645,7 @@ class MockResource(ResourceImageDict): iac_resource_mock.key = "id" asset_mock = Mock(spec=ImageAsset) asset_mock.source_local_image = original_image - iac_resource_mock.assets = [asset_mock] + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_image iac_resource_mock.get.return_value = resource_dict @@ -673,10 +673,10 @@ class MockResource(ResourceZip): original_path = "/path/to/file" iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] - asset_mock = iac_resource_mock.assets[0] + asset_mock = MagicMock(spec=S3Asset) asset_mock.source_path = original_path asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_path iac_resource_mock.get.return_value = resource_dict @@ -729,10 +729,10 @@ class MockResource(ResourceZip): original_path = "/path/to/zip_file" iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] - asset_mock = iac_resource_mock.assets[0] + asset_mock = MagicMock(spec=S3Asset) asset_mock.source_path = original_path asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_path iac_resource_mock.get.return_value = resource_dict @@ -779,10 +779,10 @@ class MockResourceNoForceZip(ResourceZip): original_path = "/path/to/file" iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] - asset_mock = iac_resource_mock.assets[0] + asset_mock = MagicMock(spec=S3Asset) asset_mock.source_path = original_path asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = original_path iac_resource_mock.get.return_value = resource_dict @@ -817,10 +817,10 @@ class MockResource(ResourceZip): iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] - asset_mock = iac_resource_mock.assets[0] + asset_mock = MagicMock(spec=S3Asset) asset_mock.source_path = None asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" @@ -846,10 +846,9 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = None + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" @@ -869,7 +868,7 @@ class MockResource(ResourceZip): resource = MockResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) iac_resource_mock = MagicMock(spec=IacResource) - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] + iac_resource_mock.find_asset_by_source_property.return_value = MagicMock(spec=S3Asset) iac_resource_mock.key = "id" parent_dir = "dir" s3_url = "s3://foo/bar" @@ -901,11 +900,10 @@ class MockResource(ResourceWithS3UrlDict): # Case 1: Property value is a path to file iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = "/path/to/file" asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {} resource_dict[resource.PROPERTY_NAME] = "/path/to/file" iac_resource_mock.get.return_value = resource_dict @@ -938,10 +936,10 @@ class MockResource(ResourceZip): iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = [MagicMock(spec=S3Asset)] - asset_mock = iac_resource_mock.assets[0] + asset_mock = MagicMock(spec=S3Asset) asset_mock.source_path = "/path/to/file" asset_mock.source_property = resource.PROPERTY_NAME + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {resource.PROPERTY_NAME: "/path/to/file"} iac_resource_mock.get.return_value = resource_dict parent_dir = "dir" @@ -974,11 +972,10 @@ def test_export_cloudformation_stack(self, TemplateMock): nested_stack_mock = Mock() iac_resource_mock.nested_stack = nested_stack_mock - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = template_path asset_mock.source_property = property_name + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock stack_resource.export(iac_resource_mock, parent_dir) self.assertEqual(resource_dict[property_name], result_path_style_s3_url) @@ -1077,11 +1074,10 @@ def test_export_cloudformation_stack_no_upload_path_not_file(self): # Case 3: Path is not a file with self.make_temp_dir() as dirname: - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=S3Asset) asset_mock.source_path = dirname asset_mock.source_property = property_name + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock resource_dict = {property_name: dirname} iac_resource_mock.get.return_value = resource_dict with self.assertRaises(exceptions.ExportFailedError): @@ -1095,9 +1091,8 @@ def test_export_serverless_application(self, TemplateMock): iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" property_name = stack_resource.PROPERTY_NAME - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=S3Asset) + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock exported_template_dict = {"foo": "bar"} result_s3_url = "s3://hello/world" result_path_style_s3_url = "http://s3.amazonws.com/hello/world" @@ -1181,9 +1176,8 @@ def test_export_serverless_application_no_upload_path_not_file(self): stack_resource = ServerlessApplicationResource(self.uploaders_mock, self.code_signer_mock, self.iac_mock) iac_resource_mock = MagicMock(spec=IacResource) iac_resource_mock.key = "id" - iac_resource_mock.assets = MagicMock() - iac_resource_mock.assets[0] = Mock() - asset_mock = iac_resource_mock.assets[0] + asset_mock = Mock(spec=S3Asset) + iac_resource_mock.find_asset_by_source_property.return_value = asset_mock property_name = stack_resource.PROPERTY_NAME # Case 3: Path is not a file From 8098e888b01bf65b38aa66d094e51e776be5a6bc Mon Sep 17 00:00:00 2001 From: Wing Fung Lau <4760060+hawflau@users.noreply.github.com> Date: Mon, 12 Jul 2021 10:38:49 -0700 Subject: [PATCH 33/38] Fix Handling of Metadata section --- samcli/lib/iac/cfn_iac.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samcli/lib/iac/cfn_iac.py b/samcli/lib/iac/cfn_iac.py index 8af89005f5..023037f463 100644 --- a/samcli/lib/iac/cfn_iac.py +++ b/samcli/lib/iac/cfn_iac.py @@ -16,6 +16,7 @@ ) from samcli.lib.utils.packagetype import IMAGE, ZIP from samcli.lib.iac.interface import ( + DictSectionItem, IacPlugin, ImageAsset, Project, @@ -104,6 +105,8 @@ def _build_stack(self, path: str, is_nested: bool = False, name: Optional[str] = metadata_section = stack.get("Metadata", DictSection()) for metadata in metadata_section.section_items: + if not isinstance(metadata, DictSectionItem): + continue metadata_type = metadata.item_id metadata_body = metadata.body metadata_assets = [] From c553eac5cb5bfb68011171355c5960347f4a3567 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar Date: Mon, 12 Jul 2021 13:55:52 -0700 Subject: [PATCH 34/38] enable integration test for python 3.6 --- appveyor.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 724081f1c3..b36da22684 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,15 +9,15 @@ environment: matrix: - # - PYTHON_HOME: "C:\\Python36-x64" - # PYTHON_VERSION: '3.6' - # PYTHON_ARCH: '64' - # NOSE_PARAMETERIZED_NO_WARN: 1 - # INSTALL_PY_37_PIP: 1 - # INSTALL_PY_38_PIP: 1 - # AWS_S3: 'AWS_S3_36' - # AWS_ECR: 'AWS_ECR_36' - # APPVEYOR_CONSOLE_DISABLE_PTY: true + - PYTHON_HOME: "C:\\Python36-x64" + PYTHON_VERSION: '3.6' + PYTHON_ARCH: '64' + NOSE_PARAMETERIZED_NO_WARN: 1 + INSTALL_PY_37_PIP: 1 + INSTALL_PY_38_PIP: 1 + AWS_S3: 'AWS_S3_36' + AWS_ECR: 'AWS_ECR_36' + APPVEYOR_CONSOLE_DISABLE_PTY: true # - PYTHON_HOME: "C:\\Python37-x64" # PYTHON_VERSION: '3.7' @@ -30,16 +30,16 @@ environment: # AWS_ECR: 'AWS_ECR_37' # APPVEYOR_CONSOLE_DISABLE_PTY: true - - PYTHON_HOME: "C:\\Python38-x64" - PYTHON_VERSION: '3.8' - PYTHON_ARCH: '64' - RUN_SMOKE: 1 - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_36_PIP: 1 - INSTALL_PY_37_PIP: 1 - AWS_S3: 'AWS_S3_38' - AWS_ECR: 'AWS_ECR_38' - APPVEYOR_CONSOLE_DISABLE_PTY: true + #- PYTHON_HOME: "C:\\Python38-x64" + # PYTHON_VERSION: '3.8' + # PYTHON_ARCH: '64' + # RUN_SMOKE: 1 + # NOSE_PARAMETERIZED_NO_WARN: 1 + # INSTALL_PY_36_PIP: 1 + # INSTALL_PY_37_PIP: 1 + # AWS_S3: 'AWS_S3_38' + # AWS_ECR: 'AWS_ECR_38' + # APPVEYOR_CONSOLE_DISABLE_PTY: true for: - From c6e9ae8fda6ef8d123bec60220f525b85ddf6ea6 Mon Sep 17 00:00:00 2001 From: Mohamed Elasmar Date: Mon, 12 Jul 2021 16:07:48 -0700 Subject: [PATCH 35/38] enable integration test for python 3.7 --- appveyor.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index b36da22684..6d6a7d2630 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,27 +9,27 @@ environment: matrix: - - PYTHON_HOME: "C:\\Python36-x64" - PYTHON_VERSION: '3.6' - PYTHON_ARCH: '64' - NOSE_PARAMETERIZED_NO_WARN: 1 - INSTALL_PY_37_PIP: 1 - INSTALL_PY_38_PIP: 1 - AWS_S3: 'AWS_S3_36' - AWS_ECR: 'AWS_ECR_36' - APPVEYOR_CONSOLE_DISABLE_PTY: true - - # - PYTHON_HOME: "C:\\Python37-x64" - # PYTHON_VERSION: '3.7' + # - PYTHON_HOME: "C:\\Python36-x64" + # PYTHON_VERSION: '3.6' # PYTHON_ARCH: '64' - # RUN_SMOKE: 1 # NOSE_PARAMETERIZED_NO_WARN: 1 - # INSTALL_PY_36_PIP: 1 + # INSTALL_PY_37_PIP: 1 # INSTALL_PY_38_PIP: 1 - # AWS_S3: 'AWS_S3_37' - # AWS_ECR: 'AWS_ECR_37' + # AWS_S3: 'AWS_S3_36' + # AWS_ECR: 'AWS_ECR_36' # APPVEYOR_CONSOLE_DISABLE_PTY: true + - PYTHON_HOME: "C:\\Python37-x64" + PYTHON_VERSION: '3.7' + PYTHON_ARCH: '64' + RUN_SMOKE: 1 + NOSE_PARAMETERIZED_NO_WARN: 1 + INSTALL_PY_36_PIP: 1 + INSTALL_PY_38_PIP: 1 + AWS_S3: 'AWS_S3_37' + AWS_ECR: 'AWS_ECR_37' + APPVEYOR_CONSOLE_DISABLE_PTY: true + #- PYTHON_HOME: "C:\\Python38-x64" # PYTHON_VERSION: '3.8' # PYTHON_ARCH: '64' From 6b3060599792ea872719480d4fb2994cd6f53d13 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 13 Jul 2021 07:56:24 -0700 Subject: [PATCH 36/38] kick off tests From 3afa48d7b5f6b0179f84991ac70e833d70deca6f Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 13 Jul 2021 19:15:41 -0700 Subject: [PATCH 37/38] fix: interactive creating CDK project won't direct to the correct resource (#3044) --- samcli/commands/init/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samcli/commands/init/__init__.py b/samcli/commands/init/__init__.py index 626a9996bc..8eabc705f7 100644 --- a/samcli/commands/init/__init__.py +++ b/samcli/commands/init/__init__.py @@ -290,7 +290,8 @@ def do_cli( from samcli.commands.exceptions import LambdaImagesTemplateException _deprecate_notification(runtime) - + if project_type: + project_type = ProjectTypes(project_type) # check for required parameters zip_bool = name and runtime and dependency_manager and app_template image_bool = name and pt_explicit and base_image From 37a1a95d829752f42cb08bc1bd7f6756660921b5 Mon Sep 17 00:00:00 2001 From: Raymond Wang <14915548+wchengru@users.noreply.github.com> Date: Tue, 13 Jul 2021 20:15:55 -0700 Subject: [PATCH 38/38] chore: bump cdk beta version --- samcli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samcli/__init__.py b/samcli/__init__.py index 2d0372606f..166aea32fd 100644 --- a/samcli/__init__.py +++ b/samcli/__init__.py @@ -2,4 +2,4 @@ SAM CLI version """ -__version__ = "1.22.0.dev202104291816" +__version__ = "1.22.0.dev202107140310"