Merge branch 'master' into enable-enterprise-support-on-creation

docs/admin-guide.md

@@ -249,6 +249,8 @@ Along with Pipeline Parameters there can potentially be stage parameters if requ
           object_key: input.zip
       deploy:
         provider: s3
+    tags:
+      owner: john
     targets:
       - path: 222222222222
         regions: eu-west-1

docs/user-guide.md

@@ -36,6 +36,8 @@ pipelines:
           account_id: 111112233332 # The AWS Account where the source code will be in a CodeCommit Repository
     params:
         notification_endpoint: [email protected] # Optional
+    tags:
+      foo: bar # Pipelines support tagging
     targets:
       - path: /security
         regions: eu-west-1
@@ -442,7 +444,7 @@ Syntax:
 upload:${style}:${local_path}
 
 # Or, when you would like to choose a specific region:
-upload:${style}:${region}:${local_path}
+upload:${region}:${style}:${local_path}
 ```
 
 There are five different styles that one could choose from.
@@ -470,7 +472,7 @@ There are five different styles that one could choose from.
 The `region` is optional.
 This allows you to upload files to S3 Buckets within specific regions by
 adding in the region name as part of the string
-(eg. `upload:path:us-west-1:productY/template.yml`).
+(eg. `upload:us-west-1:path:productY/template.yml`).
 
 The `local_path` references the files that you would like to be uploaded from
 the location where `adf-build/generate-params.py` scripts gets executed from.

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml

@@ -658,7 +658,7 @@ Resources:
                 nodejs: 10
             pre_build:
               commands:
-                - npm install cdk@1.20 -g -y --quiet --no-progress
+                - npm install cdk@1.32 -g -y --quiet --no-progress
                 - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet
                 - pip install -r adf-build/requirements.txt -q -t ./adf-build
             build:

src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml

@@ -19,6 +19,7 @@ Resources:
             Sid: "CloudFormation"
             Action: # These below actions are examples, change these to your requirements..
               - "apigateway:*"
+              - "cloudformation:*" # You will need Cloudformation actions in order to work with Cloudformation
               - "ecr:*"
               - "ecs:*"
               - "ec2:*"

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/provisioner/main.py

@@ -83,8 +83,9 @@ def create_or_update_account(org_session, support_session, account, adf_role_nam
             for _ in executor.map(lambda f: schedule_delete_default_vpc(*f), args):
                 pass
 
-    LOGGER.info(f'Ensuring account alias for {account_id} of {account.alias}')
-    org_session.create_account_alias(account.alias, role)
+    if account.alias:
+        LOGGER.info(f'Ensuring account alias for {account_id} of {account.alias}')
+        org_session.create_account_alias(account.alias, role)
 
     if account.tags:
         LOGGER.info(f'Ensuring tags exist for account {account_id}: {account.tags}')

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/provisioner/src/account.py

@@ -26,13 +26,9 @@ def __init__(
         self.delete_default_vpc = delete_default_vpc
         self.allow_direct_move_between_ou = allow_direct_move_between_ou
         self.allow_billing = allow_billing
+        self.alias = alias
         self.support_level = support_level
 
-        if alias is None:
-            self.alias = full_name
-        else:
-            self.alias = alias
-
         if tags is None:
             self.tags = {}
         else:

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/provisioner/src/configparser.py

@@ -20,4 +20,5 @@ def read_config_files(folder):
                 config = yaml.safe_load(stream)
                 for account in config.get('accounts', []):
                     accounts.append(Account.load_from_config(account))
+
     return accounts

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt

@@ -1,10 +1,10 @@
 # Install libs here that you might want in AWS CodeBuild (On Master Account)
-boto3~=1.10, >=1.10.47
+boto3==1.12.46
 pylint~=2.2.2
 pytest~=3.0.7
 mock~=2.0.0
 pyyaml~=5.1
 astroid~=2.1.0
 six~=1.11.0
-aws-sam-cli==0.22.0
+aws-sam-cli==0.46.2
 pip==19.1.1

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py

@@ -36,20 +36,18 @@ def __init__(self, **kwargs):
         self.action_name = kwargs.get('action_name')
         self.action_mode = kwargs.get('action_mode', '').upper()
         self.region = kwargs.get('region') or ADF_DEPLOYMENT_REGION
-        self.account_id = self.map_params["default_providers"]["source"].get('property', {}).get("account_id")
+        self.account_id = self.map_params["default_providers"]["source"].get('properties', {}).get("account_id")
         self.role_arn = self._generate_role_arn()
         self.notification_endpoint = self.map_params.get("topic_arn")
         self.configuration = self._generate_configuration()
         self.config = self.generate()
 
     def _generate_role_arn(self):
-        if self.target.get('properties', {}).get('role') or self.map_params["default_providers"]["build"].get('properties', {}).get("role"):
-            if self.provider == 'CodeBuild' and self.category == 'Build':
-                # CodePipeline would need access to assume this if you pass in a custom role
-                return 'arn:aws:iam::{0}:role/{1}'.format(self.account_id, self.map_params["default_providers"]["build"].get('properties', {}).get("role"))
-        if self.target.get('properties', {}).get('role') or self.map_params["default_providers"]["deploy"].get('properties', {}).get("role"):
-            if self.category == 'Deploy':
-                return 'arn:aws:iam::{0}:role/{1}'.format(self.account_id, self.map_params["default_providers"]["deploy"].get('properties', {}).get("role"))
+        default_provider = self.map_params['default_providers'][self.category.lower()]
+        specific_role = self.target.get('properties', {}).get('role') or default_provider.get('properties', {}).get('role')
+        if specific_role:
+            account_id = self.account_id if self.provider == 'CodeBuild' else self.target['id']
+            return 'arn:aws:iam::{0}:role/{1}'.format(account_id, specific_role)
         return None
 
     def _generate_configuration(self): #pylint: disable=R0912, R0911, R0915
@@ -347,7 +345,8 @@ def __init__(self, scope: core.Construct, id: str, map_params: dict, ssm_params:
             "restart_execution_on_update": map_params.get('params', {}).get('restart_execution_on_update', False),
             "name": "{0}{1}".format(ADF_PIPELINE_PREFIX, map_params['name']),
             "stages": stages,
-            "artifact_stores": Pipeline.generate_artifact_stores(map_params, ssm_params)
+            "artifact_stores": Pipeline.generate_artifact_stores(map_params, ssm_params),
+            "tags": Pipeline.restructure_tags(map_params.get('tags', {}))
         }
         self.cfn = _codepipeline.CfnPipeline(
             self,
@@ -374,6 +373,13 @@ def __init__(self, scope: core.Construct, id: str, map_params: dict, ssm_params:
             }
         })
 
+    @staticmethod
+    def restructure_tags(current_tags):
+        tags = []
+        for k, v in current_tags.items():
+            tags.append({"key": k, "value": v})
+        return tags
+
     @staticmethod
     def generate_artifact_stores(map_params, ssm_params):
         output = []

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/pipeline.py

@@ -21,6 +21,7 @@ def __init__(self, pipeline):
         self.stage_regions = []
         self.top_level_regions = pipeline.get('regions', [])
         self.completion_trigger = pipeline.get('completion_trigger', {})
+        self.tags = pipeline.get('tags', {})
         self.schedule = self.parameters.get('schedule', {})
         if not isinstance(self.completion_trigger.get('pipelines', []), list):
             self.completion_trigger['pipelines'] = [self.completion_trigger['pipelines']]
@@ -62,6 +63,7 @@ def generate_input(self):
             "environments": self.template_dictionary,
             "name": self.name,
             "params": self.parameters,
+            "tags": self.tags,
             "default_providers": self.default_providers,
             "top_level_regions": sorted(self.flatten_list(list(set(self.top_level_regions)))),
             "regions": sorted(list(set(self.flatten_list(self.stage_regions)))),

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py

@@ -347,6 +347,6 @@ def create_account(self, account, adf_role_name):
             sleep(5)  # waiting for 5 sec before checking account status again
         account_id = response["AccountId"]
         # TODO: Instead of sleeping, query for the role.
-        sleep(20)  # Wait until OrganizationalRole is created in new account
+        sleep(90)  # Wait 90 sec until OrganizationalRole is created in new account (Temp solution)
 
         return account_id

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt

@@ -4,53 +4,53 @@ mock==2.0.0
 boto3~=1.10, >=1.10.47
 pyyaml>=5.1
 schema==0.7.1
-jsii~=0.21.1
-aws_cdk.aws_codepipeline==1.20
-aws_cdk.aws_lambda_event_sources==1.20
-aws-cdk.aws-codepipeline-actions==1.20
-aws_cdk.aws_codebuild==1.20
-aws_cdk.aws_codecommit==1.20
-aws-cdk.aws-s3==1.20
-aws-cdk-assets==1.20
-aws-cdk.aws-iam==1.20
-aws-cdk.aws-kms==1.20
-aws-cdk.aws-lambda==1.20
-aws-cdk.aws-sns==1.20
-aws-cdk.aws_secretsmanager==1.20
-aws-cdk.aws_ssm==1.20
-aws-cdk.aws_events==1.20
-aws_cdk.aws_cloudformation==1.20
-aws_cdk.aws_certificatemanager==1.20
-aws_cdk.aws_sqs==1.20
-aws_cdk.aws_ec2==1.20
-aws_cdk.aws_sns==1.20
-aws_cdk.aws_elasticloadbalancing==1.20
-aws_cdk.aws_s3_notifications==1.20
-aws_cdk.aws_codedeploy==1.20
-aws_cdk.aws_ecr==1.20
-aws_cdk.aws_stepfunctions==1.20
-aws_cdk.aws_servicediscovery==1.20
-aws_cdk.aws_autoscaling_hooktargets==1.20
-aws_cdk.aws_cloudfront==1.20
-aws_cdk.aws_route53_targets==1.20
-aws_cdk.aws_route53==1.20
-aws_cdk.aws_ecs==1.20
-aws_cdk.aws_events_targets==1.20
-aws_cdk.aws_dynamodb==1.20
-aws_cdk.aws_applicationautoscaling==1.20
-aws_cdk.aws_autoscaling==1.20
-aws_cdk.aws_autoscaling_common==1.20
-aws_cdk.aws_s3_assets==1.20
-aws_cdk.aws_ecr_assets==1.20
-aws_cdk.aws_elasticloadbalancingv2==1.20
-aws_cdk.aws_apigateway==1.20
-aws_cdk.aws_logs==1.20
-aws_cdk.aws_ecr==1.20
-aws_cdk.aws_cloudwatch==1.20
-aws_cdk.aws_kinesis==1.20
-aws_cdk.aws_s3_assets==1.20
-aws_cdk.aws_sns_subscriptions==1.20
-aws-cdk.aws-sam==1.20
-aws-cdk.cx-api==1.20
-aws-cdk.region-info==1.20
-aws-cdk.core==1.20
+jsii==1.1.0
+aws_cdk.aws_codepipeline==1.32
+aws_cdk.aws_lambda_event_sources==1.32
+aws-cdk.aws-codepipeline-actions==1.32
+aws_cdk.aws_codebuild==1.32
+aws_cdk.aws_codecommit==1.32
+aws-cdk.aws-s3==1.32
+aws-cdk-assets==1.32
+aws-cdk.aws-iam==1.32
+aws-cdk.aws-kms==1.32
+aws-cdk.aws-lambda==1.32
+aws-cdk.aws-sns==1.32
+aws-cdk.aws_secretsmanager==1.32
+aws-cdk.aws_ssm==1.32
+aws-cdk.aws_events==1.32
+aws_cdk.aws_cloudformation==1.32
+aws_cdk.aws_certificatemanager==1.32
+aws_cdk.aws_sqs==1.32
+aws_cdk.aws_ec2==1.32
+aws_cdk.aws_sns==1.32
+aws_cdk.aws_elasticloadbalancing==1.32
+aws_cdk.aws_s3_notifications==1.32
+aws_cdk.aws_codedeploy==1.32
+aws_cdk.aws_ecr==1.32
+aws_cdk.aws_stepfunctions==1.32
+aws_cdk.aws_servicediscovery==1.32
+aws_cdk.aws_autoscaling_hooktargets==1.32
+aws_cdk.aws_cloudfront==1.32
+aws_cdk.aws_route53_targets==1.32
+aws_cdk.aws_route53==1.32
+aws_cdk.aws_ecs==1.32
+aws_cdk.aws_events_targets==1.32
+aws_cdk.aws_dynamodb==1.32
+aws_cdk.aws_applicationautoscaling==1.32
+aws_cdk.aws_autoscaling==1.32
+aws_cdk.aws_autoscaling_common==1.32
+aws_cdk.aws_s3_assets==1.32
+aws_cdk.aws_ecr_assets==1.32
+aws_cdk.aws_elasticloadbalancingv2==1.32
+aws_cdk.aws_apigateway==1.32
+aws_cdk.aws_logs==1.32
+aws_cdk.aws_ecr==1.32
+aws_cdk.aws_cloudwatch==1.32
+aws_cdk.aws_kinesis==1.32
+aws_cdk.aws_s3_assets==1.32
+aws_cdk.aws_sns_subscriptions==1.32
+aws-cdk.aws-sam==1.32
+aws-cdk.cx-api==1.32
+aws-cdk.region-info==1.32
+aws-cdk.core==1.32

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/schema_validation.py

@@ -14,7 +14,7 @@
 PARAM_SCHEMA = {
     Optional("notification_endpoint"): str,
     Optional("schedule"): str,
-    Optional("restart_execution_on_update"): bool,
+    Optional("restart_execution_on_update"): bool
 }
 
 AWS_ACCOUNT_ID_REGEX_STR = r"\A[0-9]{12}\Z"
@@ -296,6 +296,7 @@
     "name": And(str, len),
     "default_providers": PROVIDER_SCHEMA,
     Optional("params"): PARAM_SCHEMA,
+    Optional("tags"): dict,
     Optional("targets"): [Or(str, int, TARGET_SCHEMA, TARGET_LIST_SCHEMA)],
     Optional("regions"): REGION_SCHEMA,
     Optional("completion_trigger"): COMPLETION_TRIGGERS_SCHEMA

src/template.yml

@@ -14,7 +14,7 @@ Metadata:
     ReadmeUrl: ../docs/serverless-application-repo.md
     Labels: ['adf', 'aws-deployment-framework', 'multi-account', 'cicd', 'devops']
     HomePageUrl: https://github.com/awslabs/aws-deployment-framework
-    SemanticVersion: 3.0.5
+    SemanticVersion: 3.0.6
     SourceCodeUrl: https://github.com/awslabs/aws-deployment-framework
 Parameters:
   CrossAccountAccessRoleName:
@@ -172,7 +172,7 @@ Resources:
           TERMINATION_PROTECTION: false
           MASTER_ACCOUNT_ID: !Ref AWS::AccountId
           ORGANIZATION_ID: !GetAtt Organization.OrganizationId
-          ADF_VERSION: 3.0.5
+          ADF_VERSION: 3.0.6
           ADF_LOG_LEVEL: INFO
       FunctionName: StackWaiter
       Role: !GetAtt LambdaRole.Arn
@@ -193,7 +193,7 @@ Resources:
           DEPLOYMENT_ACCOUNT_BUCKET: !GetAtt SharedModulesBucketName.Value
           MASTER_ACCOUNT_ID: !Ref AWS::AccountId
           ORGANIZATION_ID: !GetAtt Organization.OrganizationId
-          ADF_VERSION: 3.0.5
+          ADF_VERSION: 3.0.6
           ADF_LOG_LEVEL: INFO
       FunctionName: DetermineEventFunction
       Role: !GetAtt LambdaRole.Arn
@@ -214,7 +214,7 @@ Resources:
           DEPLOYMENT_ACCOUNT_BUCKET: !GetAtt SharedModulesBucketName.Value
           MASTER_ACCOUNT_ID: !Ref AWS::AccountId
           ORGANIZATION_ID: !GetAtt Organization.OrganizationId
-          ADF_VERSION: 3.0.5
+          ADF_VERSION: 3.0.6
           ADF_LOG_LEVEL: INFO
       FunctionName: CrossAccountExecuteFunction
       Role: !GetAtt LambdaRole.Arn
@@ -233,7 +233,7 @@ Resources:
           S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket
           TERMINATION_PROTECTION: false
           MASTER_ACCOUNT_ID: !Ref AWS::AccountId
-          ADF_VERSION: 3.0.5
+          ADF_VERSION: 3.0.6
           ADF_LOG_LEVEL: INFO
       FunctionName: RoleStackDeploymentFunction
       Role: !GetAtt LambdaRole.Arn
@@ -252,7 +252,7 @@ Resources:
           S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket
           TERMINATION_PROTECTION: false
           MASTER_ACCOUNT_ID: !Ref AWS::AccountId
-          ADF_VERSION: 3.0.5
+          ADF_VERSION: 3.0.6
           ADF_LOG_LEVEL: INFO
       FunctionName: MovedToRootActionFunction
       Role: !GetAtt LambdaRole.Arn
@@ -271,7 +271,7 @@ Resources:
           S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket
           TERMINATION_PROTECTION: false
           MASTER_ACCOUNT_ID: !Ref AWS::AccountId
-          ADF_VERSION: 3.0.5
+          ADF_VERSION: 3.0.6
           ADF_LOG_LEVEL: INFO
       FunctionName: UpdateResourcePoliciesFunction
       Role: !GetAtt LambdaRole.Arn
@@ -449,7 +449,7 @@ Resources:
         Image: "aws/codebuild/standard:2.0"
         EnvironmentVariables:
           - Name: ADF_VERSION
-            Value: 3.0.5
+            Value: 3.0.6
           - Name: TERMINATION_PROTECTION
             Value: false
           - Name: PYTHONPATH
@@ -714,7 +714,7 @@ Resources:
     Type: Custom::InitialCommit
     Properties:
       ServiceToken: !GetAtt InitialCommitHandler.Arn
-      Version: 3.0.5
+      Version: 3.0.6
       RepositoryArn: !GetAtt CodeCommitRepository.Arn
       DirectoryName: bootstrap_repository
       ExistingAccountId: !Ref DeploymentAccountId
@@ -935,7 +935,7 @@ Resources:
           Id: adf-codepipeline-trigger-bootstrap
 Outputs:
   ADFVersionNumber:
-    Value: 3.0.5
+    Value: 3.0.6
     Export:
       Name: "ADFVersionNumber"
   LayerArn:

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py38
+envlist = py37
 skipsdist = True