From 2099f36a3719de827977749fbaec534137b96b6b Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Wed, 21 Apr 2021 18:45:10 +0200 Subject: [PATCH] Add ability to disable trigger on changes for source providers **Why?** For several use cases, it would be great to have a pipeline that would not trigger on source updates. For example, a pipeline might only need to be triggered on a schedule, by the completion of another pipeline, or manually. This was requested by the following issues: * Issue #308 - Starting pipeline only on completion_trigger * Issue #337 - Question: Disable webhook for Github provider **What?** The CodeCommit, Github, and S3 source providers have been update to support disabling triggering on source changes. As described in the `docs/providers-guide.md` documentation, you can optionally disable it when required. This code also solves a bug when a pipeline would rely on polling of changes for CodeCommit. In the previous version, it would be triggering twice instead of once when the source account id was specified. As the event handler would be listening for events that indicate an update was performed, even when the pipeline was set to use polling for changes instead of events. --- docs/providers-guide.md | 32 +++++++++++++++++++ .../cdk/cdk_constructs/adf_codepipeline.py | 12 +++++-- .../shared/cdk/cdk_constructs/adf_events.py | 8 ++++- .../shared/cdk/cdk_constructs/adf_github.py | 7 +++- .../cdk/cdk_stacks/adf_default_pipeline.py | 2 +- .../adf-build/shared/schema_validation.py | 9 ++++-- 6 files changed, 61 insertions(+), 9 deletions(-) diff --git a/docs/providers-guide.md b/docs/providers-guide.md index db97e3b7b..2017790d7 100644 --- a/docs/providers-guide.md +++ b/docs/providers-guide.md @@ -76,6 +76,17 @@ Provider type: `codecommit`. > The role to use to fetch the contents of the CodeCommit repository. > Only specify when you need a specific role to access it. By default ADF > will use its own role to access it instead. +- *trigger_on_changes* - *(Boolean)* default: `True`. + > Whether CodePipeline should release a change and trigger the pipeline. + > When set to False, you either need to trigger the pipeline manually, + > through a schedule, or through the completion of another pipeline. + > + > This disables the triggering of changes all together when set to False. + > In other words, when you don't want to rely on polling or event + > based triggers of changes pushed into the repository. + > + > By default, it will trigger on changes using the event triggered by + > CodeCommit when an update to the repository took place. ### GitHub @@ -104,6 +115,17 @@ Provider type: `github`. - *json_field* - *(String)* **(required)** > The name of the JSON key in the object that is stored in AWS Secrets > Manager that holds the OAuth Token. +- *trigger_on_changes* - *(Boolean)* default: `True`. + > Whether CodePipeline should release a change and trigger the pipeline. + > When set to False, you either need to trigger the pipeline manually, + > through a schedule, or through the completion of another pipeline. + > + > This disables the triggering of changes when set to False. + > It will not deploy the web hook that GitHub would otherwise use to + > trigger the pipeline on changes. + > + > By default, it will trigger deploy the web hook and trigger on changes + > using web hook call executed by GitHub. ### S3 @@ -126,6 +148,16 @@ Provider type: `s3`. - *object_key* - *(String)* **(required)** > The Specific Object within the bucket that will trigger the pipeline > execution. +- *trigger_on_changes* - *(Boolean)* default: `True`. + > Whether CodePipeline should release a change and trigger the pipeline + > if a change was detected in the S3 object. + > + > When set to False, you either need to trigger the pipeline manually, + > through a schedule, or through the completion of another pipeline. + > + > By default, it will trigger on changes using the polling mechanism + > of CodePipeline. Monitoring the S3 object so it can trigger a release + > when an update took place. ### CodeStar diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py index 7741a55cd..cf1eb739a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py @@ -69,7 +69,8 @@ def _generate_configuration(self): #pylint: disable=R0912, R0911, R0915 if self.provider == "S3" and self.category == "Source": return { "S3Bucket": self.map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('bucket_name'), - "S3ObjectKey": self.map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('object_key') + "S3ObjectKey": self.map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('object_key'), + "PollForSourceChanges": self.map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('trigger_on_changes', True), } if self.provider == "S3" and self.category == "Deploy": return { @@ -249,7 +250,10 @@ def _generate_configuration(self): #pylint: disable=R0912, R0911, R0915 return { "BranchName": self.map_params['default_providers']['source'].get('properties', {}).get('branch', 'master'), "RepositoryName": self.map_params['default_providers']['source'].get('properties', {}).get('repository', {}) or self.map_params['name'], - "PollForSourceChanges": self.map_params['default_providers']['source'].get('properties', {}).get('poll_for_changes', False) + "PollForSourceChanges": ( + self.map_params['default_providers']['source'].get('properties', {}).get('trigger_on_changes', True) + and self.map_params['default_providers']['source'].get('properties', {}).get('poll_for_changes', False) + ) } raise Exception("{0} is not a valid provider".format(self.provider)) @@ -424,7 +428,9 @@ def __init__(self, scope: core.Construct, id: str, map_params: dict, ssm_params: "provider": map_params.get('default_providers', {}).get('source', {}).get('provider'), "account_id": map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('account_id'), "repo_name": map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('repository') or map_params['name'], - "branch": map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('branch', 'master') + "branch": map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('branch', 'master'), + "poll_for_changes": map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('poll_for_changes', False), + "trigger_on_changes": map_params.get('default_providers', {}).get('source', {}).get('properties', {}).get('trigger_on_changes', True), } }) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_events.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_events.py index 2cf96fc86..76092e452 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_events.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_events.py @@ -26,7 +26,13 @@ def __init__(self, scope: core.Construct, id: str, params: dict, **kwargs): #pyl _pipeline = _codepipeline.Pipeline.from_pipeline_arn(self, 'pipeline', params["pipeline"]) _source_account = params.get('source', {}).get('account_id') _provider = params.get('source', {}).get('provider') - if _source_account and _provider == 'codecommit': + _add_trigger_on_changes = ( + _provider == 'codecommit' + and _source_account + and params.get('source', {}).get('trigger_on_changes') + and not params.get('source', {}).get('poll_for_changes') + ) + if _add_trigger_on_changes: _event = _events.Rule( self, 'trigger_{0}'.format(params["name"]), diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py index 77175b792..ddeb06324 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py @@ -37,7 +37,12 @@ def __init__(self, scope: core.Construct, id: str, map_params: dict, **kwargs): ) @staticmethod - def create_webhook(scope, pipeline, map_params): + def create_webhook_when_required(scope, pipeline, map_params): + trigger_on_changes = map_params.get("default_providers", {}).get( + "source", {}).get("properties", {}).get("trigger_on_changes", True) + if not trigger_on_changes: + return + _version = pipeline.get_att('Version') _codepipeline.CfnWebhook( scope, diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/adf_default_pipeline.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/adf_default_pipeline.py index 61ed05670..6ca6889c3 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/adf_default_pipeline.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/adf_default_pipeline.py @@ -42,7 +42,7 @@ def generate_adf_default_pipeline(scope: core.Stack, stack_input): scope, "code_pipeline", stack_input["input"], stack_input["ssm_params"], _stages ) if "github" in _source_name: - adf_github.GitHub.create_webhook(scope, _pipeline.cfn, stack_input["input"]) + adf_github.GitHub.create_webhook_when_required(scope, _pipeline.cfn, stack_input["input"]) def generate_source_stage_for_pipeline(_stages, scope, stack_input): diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/schema_validation.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/schema_validation.py index 71c70d033..d4ab0306f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/schema_validation.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/schema_validation.py @@ -49,7 +49,8 @@ Optional("branch"): str, Optional("poll_for_changes"): bool, Optional("owner"): str, - Optional("role"): str + Optional("role"): str, + Optional("trigger_on_changes"): bool, } CODECOMMIT_SOURCE = { "provider": 'codecommit', @@ -62,7 +63,8 @@ Optional("branch"): str, "owner": str, "oauth_token_path": str, - "json_field": str + "json_field": str, + Optional("trigger_on_changes"): bool, } GITHUB_SOURCE = { "provider": 'github', @@ -86,7 +88,8 @@ S3_SOURCE_PROPS = { "account_id": AWS_ACCOUNT_ID_SCHEMA, "bucket_name": str, - "object_key": str + "object_key": str, + Optional("trigger_on_changes"): bool, } S3_SOURCE = { "provider": 's3',