Merge pull request #161 from villasv/develop

Release v6 Elegant Echo

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 88

.github/workflows/push.yaml

@@ -13,12 +13,13 @@ jobs:
           submodules: true
       - name: Setup Python
         uses: actions/setup-python@v1
-      - name: Install CFN Lint
-        run: pip install cfn-lint
+      - name: Install Dev Dependencies
+        run: pip install -r dev-requirements.txt
       - name: Lint Templates
         run: make lint
 
   test:
+    if: contains(github.ref, 'refs/heads/master')
     runs-on: ubuntu-latest
     needs: lint
     steps:
@@ -28,8 +29,8 @@ jobs:
           submodules: true
       - name: Setup Python
         uses: actions/setup-python@v1
-      - name: Install TaskCat
-        run: pip install git+git://github.com/villasv/taskcat.git@b1011e8f080bad5d0a7cec65559e3c160787d17f#egg=taskcat
+      - name: Install Dev Dependencies
+        run: pip install -r dev-requirements.txt
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:
@@ -40,6 +41,7 @@ jobs:
         run: make test
 
   sync:
+    if: contains(github.ref, 'refs/heads/master')
     runs-on: ubuntu-latest
     needs: test
     steps:
@@ -49,8 +51,8 @@ jobs:
           submodules: true
       - name: Setup Python
         uses: actions/setup-python@v1
-      - name: Install AWS CLI
-        run: pip install awscli
+      - name: Install Dev Dependencies
+        run: pip install -r dev-requirements.txt
       - name: Configure AWS Credentials
         uses: aws-actions/configure-aws-credentials@v1
         with:

.github/workflows/review.yaml

@@ -0,0 +1,69 @@
+name: Stack Release Pipeline
+
+on:
+  pull_request_review:
+    types:
+      - submitted
+
+
+jobs:
+
+  lint:
+    if: contains(github.event.review.body, '/lint')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Setup Python
+        uses: actions/setup-python@v1
+      - name: Install CFN Lint
+        run: pip install cfn-lint
+      - name: Lint Templates
+        run: make lint
+
+  test:
+    if: contains(github.event.review.body, '/test')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Setup Python
+        uses: actions/setup-python@v1
+      - name: Install TaskCat
+        run: pip install git+git://github.com/villasv/taskcat.git@b1011e8f080bad5d0a7cec65559e3c160787d17f#egg=taskcat
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Test Stacks
+        run: make test
+
+  sync:
+    if: contains(github.event.review.body, '/sync')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v1
+        with:
+          submodules: true
+      - name: Setup Python
+        uses: actions/setup-python@v1
+      - name: Install AWS CLI
+        run: pip install awscli
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Infer Branch Name
+        shell: bash
+        run: echo "::set-env name=BRANCH::${GITHUB_REF#refs/heads/}"
+      - name: Sync Files
+        run: make sync

.pylintrc

@@ -1,5 +1,6 @@
 [MESSAGES CONTROL]
 
 disable=
+    fixme,
     missing-module-docstring,
     missing-function-docstring

Makefile

@@ -1,7 +1,6 @@
 ifndef BRANCH
 BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 endif
-
 ifeq ($(BRANCH),master)
 BUCKET := s3://turbine-quickstart/quickstart-turbine-airflow
 else
@@ -10,6 +9,9 @@ endif
 
 
 lint:
+	black . --check
+	flake8 .
+	pylint **/*.py
 	cfn-lint templates/*.template
 
 nuke:
@@ -19,7 +21,9 @@ pack:
 	7z a ./functions/package.zip ./functions/*.py
 
 sync: pack
+	aws s3 rm $(BUCKET) --recursive
 	aws s3 sync --exclude '.*' --acl public-read . $(BUCKET)
 
 test: pack
+	pytest -vv
 	taskcat test run --input-file ./ci/taskcat.yaml

README.md

@@ -1,6 +1,6 @@
-<img src="img/logo.png" align="right" width="25%" />
+<img src=".github/img/logo.png" align="right" width="25%" />
 
-# Turbine [![CFN Deploy](https://img.shields.io/badge/CFN-deploy-green.svg?style=flat-square&logo=amazon-aws)](#get-it-working) [![GitHub Release](https://img.shields.io/github/release/villasv/aws-airflow-stack.svg?style=flat-square&logo=github)](https://github.com/villasv/aws-airflow-stack/releases/latest) [![Build Status](https://img.shields.io/travis/villasv/aws-airflow-stack/master.svg?style=flat-square&logo=gitlab&logoColor=white&label=taskcat)](https://scrutinizer-ci.com/g/villasv/aws-airflow-stack/build-status/master)
+# Turbine [![GitHub Release](https://img.shields.io/github/release/villasv/aws-airflow-stack.svg?style=flat-square&logo=github)](https://github.com/villasv/aws-airflow-stack/releases/latest) [![Build Status](https://img.shields.io/github/workflow/status/villasv/aws-airflow-stack/Stack%20Release%20Pipeline?style=flat-square&logo=github&logoColor=white&label=build)](https://github.com/villasv/aws-airflow-stack/actions?query=workflow%3A%22Stack+Release+Pipeline%22+branch%3Amaster) [![CFN Deploy](https://img.shields.io/badge/CFN-deploy-green.svg?style=flat-square&logo=amazon-aws)](#get-it-working)
 
 Turbine is the set of bare metals behind a simple yet complete and efficient
 Airflow setup.
@@ -13,7 +13,7 @@ configure in a few commands.
 
 ## Overview
 
-![stack diagram](/img/stack-diagram.png)
+![stack diagram](/.github/img/stack-diagram.png)
 
 The stack is composed mainly of three services: the Airflow web server, the
 Airflow scheduler, and the Airflow worker. Supporting resources include an RDS
@@ -113,7 +113,7 @@ available in the shell. Before running Airflow commands, you need to load the
 Airflow configuration:
 
 ```bash
-$ export $(xargs </etc/sysconfig/airflow)
+$ export $(xargs </etc/sysconfig/airflow.env)
 $ airflow list_dags
 ```
 

dev-requirements.txt

@@ -0,0 +1,9 @@
+git+git://github.com/PyCQA/astroid.git#egg=astroid
+awscli
+black
+cfn-flip
+cfn-lint
+flake8
+git+git://github.com/PyCQA/pylint.git#egg=pylint
+pytest
+taskcat

examples/project/airflow/dags/my_dag.py

@@ -1,22 +1,17 @@
 from datetime import datetime
 
-import airflow
-from airflow.models import DAG
+from airflow import DAG
 from airflow.operators.bash_operator import BashOperator
 
 default_args = {
-    'start_date': datetime(2019, 1, 1),
+    "start_date": datetime(2019, 1, 1),
 }
 
-dag = DAG(
-    dag_id='my_dag',
-    default_args=default_args,
-    schedule_interval='@daily',
-)
+dag = DAG(dag_id="my_dag", default_args=default_args, schedule_interval="@daily",)
 
 for i in range(5):
     task = BashOperator(
-        task_id='runme_' + str(i),
+        task_id="runme_" + str(i),
         bash_command='echo "{{ task_instance_key_str }}" && sleep 5 && echo "done"',
         dag=dag,
     )

examples/project/airflow/scripts/cdapp_start.sh

@@ -1,2 +1,8 @@
-#!/bin/sh
-systemctl start airflow
+#!/bin/bash -e
+systemctl is-enabled --quiet airflow-scheduler &&\
+    systemctl start airflow-scheduler
+systemctl is-enabled --quiet airflow-webserver &&\
+    systemctl start airflow-webserver
+systemctl is-enabled --quiet airflow-workerset &&\
+    systemctl start airflow-workerset
+exit 0

examples/project/airflow/scripts/cdapp_stop.sh

@@ -1,2 +1,8 @@
-#!/bin/sh
-systemctl stop airflow
+#!/bin/bash -e
+systemctl is-enabled --quiet airflow-scheduler &&\
+    systemctl stop airflow-scheduler
+systemctl is-enabled --quiet airflow-webserver &&\
+    systemctl stop airflow-webserver
+systemctl is-enabled --quiet airflow-workerset &&\
+    systemctl stop airflow-workerset
+exit 0

scripts/airflow-confapply.sh

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+systemctl is-enabled --quiet airflow-scheduler &&\
+    systemctl restart airflow-scheduler
+systemctl is-enabled --quiet airflow-webserver &&\
+    systemctl restart airflow-webserver
+systemctl is-enabled --quiet airflow-workerset &&\
+    systemctl restart airflow-workerset
+exit 0

scripts/airflow-heartbeat.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+if [ "$(systemctl is-active airflow)" = "deactivating" ]; then
+    aws autoscaling record-lifecycle-action-heartbeat \
+    --instance-id "$(ec2-metadata -i | awk '{print $2}')" \
+    --lifecycle-hook-name "$AWS_STACK_NAME-scaling-lfhook" \
+    --auto-scaling-group-name "$AWS_STACK_NAME-scaling-group" \
+    --region "$AWS_REGION"
+fi

scripts/airflow-terminate.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+INSTANCE_ID=$(ec2-metadata -i | awk '{print $2}')
+TERMINATE_MESSAGE="Terminating EC2 instance <$INSTANCE_ID>"
+TERMINATING=$(aws autoscaling describe-scaling-activities \
+    --auto-scaling-group-name "$ClusterStack-scaling-group" \
+    --max-items 100 \
+    --region "$AWS_REGION" | \
+    jq --arg TERMINATE_MESSAGE "$TERMINATE_MESSAGE" \
+    '.Activities[]
+    | select(.Description
+    | test($TERMINATE_MESSAGE)) != []')
+if [ "$TERMINATING" = "true" ]; then
+    systemctl stop airflow
+fi

scripts/commons.setup.sh

@@ -0,0 +1,99 @@
+#!/bin/bash -e
+yum install -y jq
+jsonvar() { jq -n --argjson doc "$1" -r "\$doc.$2"; }
+
+IMDSv1="http://169.254.169.254/latest"
+AWS_PARTITION=$(curl "$IMDSv1/meta-data/services/partition")
+export AWS_PARTITION
+
+IAM_ROLE=$(curl "$IMDSv1/meta-data/iam/security-credentials")
+IAM_DOCUMENT=$(curl "$IMDSv1/meta-data/iam/security-credentials/$IAM_ROLE")
+AWS_ACCESS_KEY_ID=$(jsonvar "$IAM_DOCUMENT" AccessKeyId)
+AWS_SECRET_ACCESS_KEY=$(jsonvar "$IAM_DOCUMENT" SecretAccessKey)
+AWS_SECURITY_TOKEN=$(jsonvar "$IAM_DOCUMENT" Token)
+export IAM_ROLE AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SECURITY_TOKEN
+
+EC2_DOCUMENT=$(curl "$IMDSv1/dynamic/instance-identity/document")
+AWS_REGION=$(jsonvar "$EC2_DOCUMENT" region)
+AWS_DEFAULT_REGION=$(jsonvar "$EC2_DOCUMENT" region)
+AWS_ACCOUNT_ID=$(jsonvar "$EC2_DOCUMENT" accountId)
+EC2_INSTANCE_ID=$(jsonvar "$EC2_DOCUMENT" instanceId)
+export AWS_DEFAULT_REGION AWS_REGION AWS_ACCOUNT_ID EC2_INSTANCE_ID
+
+yum install -y python3 python3-pip python3-wheel python3-devel
+pip3 install awscurl
+EC2_HOST_IDENTIFIER="arn:$AWS_PARTITION:ec2:$AWS_REGION:$AWS_ACCOUNT_ID"
+EC2_HOST_IDENTIFIER="$EC2_HOST_IDENTIFIER:instance/$EC2_INSTANCE_ID"
+CD_COMMAND=$(/usr/local/bin/awscurl -X POST \
+    --service codedeploy-commands \
+    "https://codedeploy-commands.$AWS_REGION.amazonaws.com" \
+    -H "X-AMZ-TARGET: CodeDeployCommandService_v20141006.PollHostCommand" \
+    -H "Content-Type: application/x-amz-json-1.1" \
+    -d "{\"HostIdentifier\": \"$EC2_HOST_IDENTIFIER\"}")
+if [ "$CD_COMMAND" = "" ] || [ "$CD_COMMAND" = "b'{}'" ]
+then CD_PENDING_DEPLOY="false"
+else CD_PENDING_DEPLOY="true"
+fi
+export CD_PENDING_DEPLOY
+
+DB_SECRETS=$(aws secretsmanager \
+    get-secret-value --secret-id "$DB_SECRETS_ARN")
+DB_ENGINE=$(jsonvar "$DB_SECRETS" "SecretString | fromjson.engine")
+DB_USER=$(jsonvar "$DB_SECRETS" "SecretString | fromjson.username")
+DB_PASS=$(jsonvar "$DB_SECRETS" "SecretString | fromjson.password")
+DB_HOST=$(jsonvar "$DB_SECRETS" "SecretString | fromjson.host")
+DB_DBNAME=$(jsonvar "$DB_SECRETS" "SecretString | fromjson.dbname")
+DB_PORT=$(jsonvar "$DB_SECRETS" "SecretString | fromjson.port")
+DATABASE_URI="$DB_ENGINE://$DB_USER:$DB_PASS@$DB_HOST:$DB_PORT/$DB_DBNAME"
+export DATABASE_URI
+
+yum install -y python3
+pip3 install cryptography
+FERNET_KEY=$(python3 -c "if True:#
+    from base64 import urlsafe_b64encode
+    from cryptography.fernet import Fernet
+    from cryptography.hazmat.backends import default_backend
+    from cryptography.hazmat.primitives import hashes
+    from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
+    kdf = PBKDF2HMAC(
+        algorithm=hashes.SHA256(),length=32,iterations=100000,
+		backend=default_backend(),salt=b'${FERNET_SALT//\'/\\\'}',
+    )
+    key = kdf.derive(b'${DB_PASS_ESC//\'/\\\'}')
+    key_encoded = urlsafe_b64encode(key)
+    print(key_encoded.decode('utf8'))")
+export FERNET_KEY
+
+FILES=$(dirname "$0")
+find "$FILES" -type f -iname "*.sh" -exec chmod +x {} \;
+envreplace() { CONTENT=$(envsubst <"$1"); echo "$CONTENT" >"$1"; }
+
+mkdir -p /etc/cfn/hooks.d
+cp "$FILES"/systemd/cfn-hup.service /lib/systemd/system/
+cp "$FILES"/systemd/cfn-hup.conf /etc/cfn/cfn-hup.conf
+cp "$FILES"/systemd/cfn-auto-reloader.conf /etc/cfn/hooks.d/cfn-auto-reloader.conf
+envreplace /etc/cfn/cfn-hup.conf
+envreplace /etc/cfn/hooks.d/cfn-auto-reloader.conf
+
+mkdir /run/airflow && chown -R ec2-user: /run/airflow
+cp "$FILES"/systemd/airflow-*.{path,timer,service} /lib/systemd/system/
+cp "$FILES"/systemd/airflow.env /etc/sysconfig/airflow.env
+cp "$FILES"/systemd/airflow.conf /usr/lib/tmpfiles.d/airflow.conf
+envreplace /etc/sysconfig/airflow.env
+
+mapfile -t AIRFLOW_ENVS < /etc/sysconfig/airflow.env
+export "${AIRFLOW_ENVS[@]}"
+
+yum install -y gcc libcurl-devel openssl-devel
+export PYCURL_SSL_LIBRARY=openssl
+pip3 install "apache-airflow[celery,postgres,s3,crypto]==1.10.9" "celery[sqs]"
+mkdir "$AIRFLOW_HOME" && chown -R ec2-user: "$AIRFLOW_HOME"
+
+systemctl enable --now cfn-hup.service
+
+cd_agent() {
+    yum install -y ruby
+    wget "https://aws-codedeploy-$AWS_REGION.s3.amazonaws.com/latest/install"
+    chmod +x ./install
+    ./install auto
+}

scripts/scheduler.setup.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -e
+
+. "$(dirname $0)/commons.setup.sh"
+
+if [ "$TURBINE__CORE__LOAD_DEFAULTS" == "True" ]; then
+    su -c '/usr/local/bin/airflow initdb' ec2-user
+else
+    su -c '/usr/local/bin/airflow upgradedb' ec2-user
+fi
+
+systemctl enable --now airflow-scheduler
+cd_agent

scripts/systemd/airflow-confapply-agent.path

@@ -0,0 +1,9 @@
+[Unit]
+After=airflow-scheduler.service airflow-webserver.service airflow-workerset.service
+PartOf=airflow-scheduler.service airflow-webserver.service airflow-workerset.service
+
+[Path]
+PathModified=/etc/sysconfig/airflow.env
+
+[Install]
+WantedBy=airflow-scheduler.service airflow-webserver.service airflow-workerset.service