chore(layers): add release pipeline in GitHub Actions (#1278)

* chore: add layer project * reduce to 1 region for dev * chore: shorter name for the workflow * fix ignore markdown lint for now * fix: more f strings * ignore mdlint * add reusable workflow for both beta and prod * Update layer/layer/canary/app.py Co-authored-by: Heitor Lessa <[email protected]> * Update layer/layer/canary/app.py Co-authored-by: Heitor Lessa <[email protected]> * readme review * rephrase canary stack ssm parameter usage * add default RELEASE_TAG_VERSION assignment based on the input (release or manual trigger) * add reference to layer docs * wording * move version trackign arn to canary stack * remove outdated npm caching, add release tag resolution for manual workflow trigger * review: fix layer name and remove dependencies from reusable workflow * remove debug statement, add default working dir * pin versions and hashes for requirements with pip-compile * rename reusable workflow * pass artefact name to the reusable workflow to prevent potential future conflicts Co-authored-by: Heitor Lessa <[email protected]>
aws-powertools · Jul 13, 2022 · 3927738 · 3927738
1 parent bb8f38e
commit 3927738
Show file tree

Hide file tree

Showing 12 changed files with 532 additions and 0 deletions.
diff --git a/.github/workflows/publish_layer.yml b/.github/workflows/publish_layer.yml
@@ -0,0 +1,79 @@
+name: Deploy layer to all regions
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      latest_published_version:
+        description: "Latest PyPi published version to rebuild latest docs for, e.g. v1.22.0"
+        default: "v1.22.0"
+        required: true
+  workflow_run:
+    workflows: [ "Publish to PyPi" ]
+    types:
+      - completed
+
+
+jobs:
+  build-layer:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./layer
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16.12'
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: 'pip'
+      - name: Set release notes tag
+        run: |
+          RELEASE_INPUT=${{ inputs.latest_published_version }}
+          GITHUB_EVENT_RELEASE_TAG=${{ github.event.release.tag_name }}
+          RELEASE_TAG_VERSION=${GITHUB_EVENT_RELEASE_TAG:-$RELEASE_INPUT}
+          echo "RELEASE_TAG_VERSION=${RELEASE_TAG_VERSION:1}" >> $GITHUB_ENV
+      - name: install cdk and deps
+        run: |
+          npm install -g [email protected]
+          cdk --version
+      - name: install deps
+        run: |
+          pip install -r requirements.txt
+      - name: CDK build
+        run: cdk synth --context version=$RELEASE_TAG_VERSION -o cdk.out
+      - name: zip output
+        run: zip -r cdk.out.zip cdk.out
+      - name: Archive CDK artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: cdk-layer-artefact
+          path: layer/cdk.out.zip
+
+  deploy-beta:
+    needs:
+      - build-layer
+    uses: ./.github/workflows/reusable_deploy_layer_stack.yml
+    with:
+      stage: "BETA"
+      artefact-name: "cdk-layer-artefact"
+    secrets:
+      target-account-role: arn:aws:iam::${{ secrets.LAYERS_BETA_ACCOUNT }}:role/${{ secrets.AWS_GITHUB_OIDC_ROLE }}
+
+  deploy-prod:
+    needs:
+      - deploy-beta
+    uses: ./.github/workflows/reusable_deploy_layer_stack.yml
+    with:
+      stage: "PROD"
+      artefact-name: "cdk-layer-artefact"
+    secrets:
+      target-account-role: arn:aws:iam::${{ secrets.LAYERS_PROD_ACCOUNT }}:role/${{ secrets.AWS_GITHUB_OIDC_ROLE }}
diff --git a/.github/workflows/reusable_deploy_layer_stack.yml b/.github/workflows/reusable_deploy_layer_stack.yml
@@ -0,0 +1,87 @@
+name: Deploy cdk stack
+
+permissions:
+  id-token: write
+  contents: read
+
+on:
+  workflow_call:
+    inputs:
+      stage:
+        required: true
+        type: string
+      artefact-name:
+        required: true
+        type: string
+    secrets:
+      target-account-role:
+        required: true
+
+jobs:
+  deploy-cdk-stack:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./layer
+    strategy:
+      fail-fast: false
+      matrix:
+        region: [
+          "af-south-1",
+          #          "eu-central-1",
+          #          "us-east-1",
+          #          "us-east-2",
+          #          "us-west-1",
+          #          "us-west-2",
+          #          "ap-east-1",
+          #          "ap-south-1",
+          #          "ap-northeast-1",
+          #          "ap-northeast-2",
+          #          "ap-southeast-1",
+          #          "ap-southeast-2",
+          #          "ca-central-1",
+          #          "eu-west-1",
+          #          "eu-west-2",
+          #          "eu-west-3",
+          #          "eu-south-1",
+          #          "eu-north-1",
+          #          "sa-east-1",
+          #          "ap-southeast-3",
+          #          "ap-northeast-3",
+          #          "me-south-1"
+        ]
+    steps:
+      - name: checkout
+        uses: actions/checkout@v2
+      - name: aws credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-region: ${{ matrix.region }}
+          role-to-assume: ${{ secrets.target-account-role }}
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '16.12'
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+          cache: 'pip'
+      - name: install cdk and deps
+        run: |
+          npm install -g [email protected]
+          cdk --version
+      - name: install deps
+        run: |
+          pip install -r requirements.txt
+      - name: Download artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ inputs.artefact-name }}
+          path: layer
+      - name: unzip artefact
+        run: unzip ${{ inputs.artefact-name }}
+      - name: CDK Deploy Layer
+        run: cdk deploy --app cdk.out --context region=${{ matrix.region }} 'LayerStack ' --require-approval never --verbose
+      - name: CDK Deploy Canary
+        run: cdk deploy --app cdk.out --context region=${{ matrix.region}} --parameters DeployStage="${{ inputs.stage }}" 'CanaryStack' --require-approval never --verbose
diff --git a/layer/.gitignore b/layer/.gitignore
@@ -0,0 +1,10 @@
+*.swp
+package-lock.json
+__pycache__
+.pytest_cache
+.venv
+*.egg-info
+
+# CDK asset staging directory
+.cdk.staging
+cdk.out
diff --git a/layer/README.md b/layer/README.md
@@ -0,0 +1,27 @@
+<!-- markdownlint-disable MD041 MD043-->
+# CDK Powertools layer
+
+This is a CDK project to build and deploy AWS Lambda Powertools [Lambda layer](https://docs.aws.amazon.com/lambda/latest/dg/gettingstarted-concepts.html#gettingstarted-concepts-layer) to multiple commercial regions.
+
+## Build the layer
+
+To build the layer construct you need to provide the Powertools version that is [available in PyPi](https://pypi.org/project/aws-lambda-powertools/).
+You can pass it as a context variable when running `synth` or `deploy`,
+
+```shell
+cdk synth --context version=1.25.1
+```
+
+## Canary stack
+
+We use a canary stack to verify that the deployment is successful and we can use the layer by adding it to a newly created Lambda function.
+The canary is deployed after the layer construct. Because the layer ARN is created during the deploy we need to pass this information async via SSM parameter.
+To achieve that we use SSM parameter store to pass the layer ARN to the canary.
+The layer stack writes the layer ARN after the deployment as SSM parameter and the canary stacks reads this information and adds the layer to the function.
+
+## Version tracking
+
+AWS Lambda versions Lambda layers by incrementing a number at the end of the ARN.
+This makes it challenging to know which Powertools version a layer contains.
+For better tracking of the ARNs and the corresponding version we need to keep track which powertools version was deployed to which layer.
+To achieve that we created two components. First, we created a version tracking app which receives events via EventBridge. Second, after a successful canary deployment we send the layer ARN, Powertools version, and the region to this EventBridge.
diff --git a/layer/app.py b/layer/app.py
@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import aws_cdk as cdk
+
+from layer.canary_stack import CanaryStack
+from layer.layer_stack import LayerStack
+
+app = cdk.App()
+
+POWERTOOLS_VERSION: str = app.node.try_get_context("version")
+SSM_PARAM_LAYER_ARN: str = "/layers/powertools-layer-arn"
+
+if not POWERTOOLS_VERSION:
+    raise ValueError(
+        "Please set the version for Powertools by passing the '--context=version:<version>' parameter to the CDK "
+        "synth step."
+    )
+
+LayerStack(app, "LayerStack", powertools_version=POWERTOOLS_VERSION, ssm_paramter_layer_arn=SSM_PARAM_LAYER_ARN)
+
+CanaryStack(app, "CanaryStack", powertools_version=POWERTOOLS_VERSION, ssm_paramter_layer_arn=SSM_PARAM_LAYER_ARN)
+
+app.synth()
diff --git a/layer/cdk.json b/layer/cdk.json
@@ -0,0 +1,35 @@
+{
+  "app": "python3 app.py",
+  "watch": {
+    "include": [
+      "**"
+    ],
+    "exclude": [
+      "README.md",
+      "cdk*.json",
+      "requirements*.txt",
+      "source.bat",
+      "**/__init__.py",
+      "python/__pycache__",
+      "tests"
+    ]
+  },
+  "context": {
+    "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true,
+    "@aws-cdk/core:stackRelativeExports": true,
+    "@aws-cdk/aws-rds:lowercaseDbIdentifier": true,
+    "@aws-cdk/aws-lambda:recognizeVersionProps": true,
+    "@aws-cdk/aws-lambda:recognizeLayerVersion": true,
+    "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true,
+    "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true,
+    "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true,
+    "@aws-cdk/core:checkSecretUsage": true,
+    "@aws-cdk/aws-iam:minimizePolicies": true,
+    "@aws-cdk/core:validateSnapshotRemovalPolicy": true,
+    "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true,
+    "@aws-cdk/core:target-partitions": [
+      "aws",
+      "aws-cn"
+    ]
+  }
+}
diff --git a/layer/layer/__init__.py b/layer/layer/__init__.py
diff --git a/layer/layer/canary/app.py b/layer/layer/canary/app.py
@@ -0,0 +1,99 @@
+import datetime
+import json
+import os
+from importlib.metadata import version
+
+import boto3
+
+from aws_lambda_powertools import Logger, Metrics, Tracer
+
+logger = Logger(service="version-track")
+tracer = Tracer()
+metrics = Metrics(namespace="powertools-layer-canary", service="PowertoolsLayerCanary")
+
+layer_arn = os.getenv("POWERTOOLS_LAYER_ARN")
+powertools_version = os.getenv("POWERTOOLS_VERSION")
+stage = os.getenv("LAYER_PIPELINE_STAGE")
+event_bus_arn = os.getenv("VERSION_TRACKING_EVENT_BUS_ARN")
+
+
+def handler(event):
+    logger.info("Running checks")
+    check_envs()
+    verify_powertools_version()
+    send_notification()
+    return True
+
+
+@logger.inject_lambda_context(log_event=True)
+def on_event(event, context):
+    request_type = event["RequestType"]
+    # we handle only create events, because we recreate the canary on each run
+    if request_type == "Create":
+        return on_create(event)
+
+    return "Nothing to be processed"
+
+
+def on_create(event):
+    props = event["ResourceProperties"]
+    logger.info("create new resource with properties %s" % props)
+    handler(event)
+
+
+def check_envs():
+    logger.info('Checking required envs ["POWERTOOLS_LAYER_ARN", "AWS_REGION", "STAGE"]')
+    if not layer_arn:
+        raise ValueError("POWERTOOLS_LAYER_ARN is not set. Aborting...")
+    if not powertools_version:
+        raise ValueError("POWERTOOLS_VERSION is not set. Aborting...")
+    if not stage:
+        raise ValueError("LAYER_PIPELINE_STAGE is not set. Aborting...")
+    if not event_bus_arn:
+        raise ValueError("VERSION_TRACKING_EVENT_BUS_ARN is not set. Aborting...")
+    logger.info("All envs configured, continue...")
+
+
+def verify_powertools_version() -> None:
+    """
+    fetches the version that we import from the powertools layer and compares
+    it with expected version set in environment variable, which we pass during deployment.
+    :raise ValueError if the expected version is not the same as the version we get from the layer
+    """
+    logger.info("Checking Powertools version in library...")
+    current_version = version("aws_lambda_powertools")
+    if powertools_version != current_version:
+        raise ValueError(
+            f'Expected powertoosl version is "{powertools_version}", but layer contains version "{current_version}"'
+        )
+    logger.info(f"Current Powertools version is: {current_version}")
+
+
+def send_notification():
+    """
+    sends an event to version tracking event bridge
+    """
+    event = {
+        "Time": datetime.datetime.now(),
+        "Source": "powertools.layer.canary",
+        "EventBusName": event_bus_arn,
+        "DetailType": "deployment",
+        "Detail": json.dumps(
+            {
+                "id": "powertools-python",
+                "stage": stage,
+                "region": os.environ["AWS_REGION"],
+                "version": powertools_version,
+                "layerArn": layer_arn,
+            }
+        ),
+    }
+
+    logger.info(f"sending notification event: {event}")
+
+    client = boto3.client("events", region_name="eu-central-1")
+    resp = client.put_events(Entries=[event])
+    logger.info(resp)
+    if resp["FailedEntryCount"] != 0:
+        logger.error(resp)
+        raise ValueError("Failed to send deployment notification to version tracking")