Merge pull request #781 from AlexsLemonade/dev

Release latest changes to master

.circleci/config.yml

@@ -4,7 +4,8 @@ jobs:
   # These are grouped together because in total they take less time than our other tests individually.
   main_tests:
     working_directory: ~/refinebio
-    machine: true
+    machine:
+      docker_layer_caching: true
     steps:
       - checkout
 
@@ -72,7 +73,8 @@ jobs:
 
   common_tests:
     working_directory: ~/refinebio
-    machine: true
+    machine:
+      docker_layer_caching: true
     steps:
       - checkout
 
@@ -96,7 +98,8 @@ jobs:
   # This tests workers tests tagged as 'salmon'
   salmon_and_api_tests:
     working_directory: ~/refinebio
-    machine: true
+    machine:
+      docker_layer_caching: true
     steps:
       - checkout
 
@@ -121,9 +124,10 @@ jobs:
           command: .circleci/filter_tests.sh -t salmon
           no_output_timeout: 1h
 
-  tx_illumina_affy_agilent_tests:
+  tx_illumina_tests:
     working_directory: ~/refinebio
-    machine: true
+    machine:
+      docker_layer_caching: true
     steps:
       - checkout
 
@@ -161,6 +165,26 @@ jobs:
       - run: docker image rm ccdlstaging/dr_transcriptome
       - run: docker image rm ccdlstaging/dr_illumina
 
+  affy_agilent_tests:
+    machine: true
+    working_directory: ~/refinebio
+    steps:
+      - checkout
+
+      # Clean up stuff circle provides that we don't need.
+      - run: sudo bash .circleci/cleanup_instance.sh
+
+      # Setup Postgres in a Container
+      - run: ./run_postgres.sh
+      # Let Postgres start up.
+      - run: sleep 30
+      # Finish setting up Postgres now that it's running.
+      - run: ./common/install_db_docker.sh
+
+      # Install our application. Provides the data_refinery_common package for the other images.
+      - run: chmod -R a+wr common
+      - run: ./update_models.sh
+
       - run:
           command: .circleci/filter_tests.sh -t affymetrix
           # This takes a while because the affymetrix image is huge
@@ -173,22 +197,16 @@ jobs:
       # This doesn't take as long because the image has already been pulled.
       - run: .circleci/filter_tests.sh -t agilent
 
-
   deploy:
     machine: true
     working_directory: ~/refinebio
     steps:
       - checkout
-
-      # Clean up stuff circle provides that we don't need.
-      - run: sudo bash .circleci/cleanup_instance.sh
-
-      - run: bash .circleci/verify_tag.sh
+      - run: bash .circleci/install_git_decrypt.sh
       - run: bash .circleci/git_decrypt.sh
-      - run: bash .circleci/update_docker_img.sh
       - run:
-          command: bash .circleci/run_terraform.sh
-          no_output_timeout: 30m # Creating a postgres instance can take up to 20 minutes.
+          command: bash .circleci/remote_deploy.sh
+          no_output_timeout: 4h # Building images and createing postgres instances can take a while.
       - run: bash .circleci/slackpost.sh robots deploybot
 
 workflows:
@@ -216,7 +234,13 @@ workflows:
             # "test" job, an explicit "tags" filter is required here.
             tags:
               only: /v.*/
-      - tx_illumina_affy_agilent_tests:
+      - tx_illumina_tests:
+          filters:
+            # To allow tag commits whose name start with "v" to trigger
+            # "test" job, an explicit "tags" filter is required here.
+            tags:
+              only: /v.*/
+      - affy_agilent_tests:
           filters:
             # To allow tag commits whose name start with "v" to trigger
             # "test" job, an explicit "tags" filter is required here.
@@ -230,7 +254,8 @@ workflows:
             - main_tests
             - common_tests
             - salmon_and_api_tests
-            - tx_illumina_affy_agilent_tests
+            - tx_illumina_tests
+            - affy_agilent_tests
           filters:
             # No branch commit will ever trigger this job.
             branches:

.circleci/git_decrypt.sh

@@ -1,12 +1,5 @@
-# Install git-crypt
-cd
-git clone https://github.com/AGWA/git-crypt.git
-cd git-crypt
-make
-sudo make install
-
 # Unlock encrypted files
 cd ~/refinebio/.circleci
-openssl aes-256-cbc -d -in exported.key.enc -out $KEY_FILENAME -k $OPENSSL_KEY
+openssl aes-256-cbc -md md5 -d -in exported.key.enc -out $KEY_FILENAME -k $OPENSSL_KEY
 git-crypt unlock $KEY_FILENAME
 rm -f $KEY_FILENAME

.circleci/install_git_decrypt.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Install git-crypt
+cd
+git clone https://github.com/AGWA/git-crypt.git
+cd git-crypt
+make
+sudo make install

.circleci/remote_deploy.sh

@@ -0,0 +1,69 @@
+#!/bin/bash -e
+
+# This script will deploy the Refinebio system using a dedicated AWS instance.
+#   First it will use that instance to build up to date Docker images
+#     and push them to Dockerhub.
+#   Next it will use terraform to update our infrastructure and restart our services.
+#   Finally it cleans up after itself.
+
+# It has been written with the intention of being run from CircleCI as
+#   part of our CI/CD process. It therefore assumes that the following
+#   environment variables will be set:
+#     - DEPLOY_IP_ADDRESS --  The IP address of the instance to run the deploy on.
+#     - CIRCLE_TAG -- The tag that was pushed to CircleCI to trigger the deploy.
+#         Will be used as the version for the system and the tag for Docker images.
+#     - DOCKER_ID -- The username that will be used to log into Dockerhub.
+#     - DOCKER_PASSWD -- The password that will be used to log into Dockerhub.
+#     - KEY_FILENNAME -- The name to use for the decrypted SSH key file.
+#     - OPENSSL_KEY -- The OpenSSl key which will be used to decrypt the SSH key.
+#     - AWS_ACCESS_KEY_ID -- The AWS key id to use when interacting with AWS.
+#     - AWS_SECRET_ACCESS_KEY -- The AWS secret key to use when interacting with AWS.
+
+
+cd ~/refinebio
+
+chmod 600 infrastructure/data-refinery-key.pem
+
+run_on_deploy_box () {
+    ssh -o StrictHostKeyChecking=no \
+        -i infrastructure/data-refinery-key.pem \
+        ubuntu@${DEPLOY_IP_ADDRESS} "cd refinebio && $1"
+}
+
+# Create file containing local env vars that are needed for deploy.
+rm -f env_vars
+echo "export CIRCLE_TAG=$CIRCLE_TAG" >> env_vars
+echo "export DOCKER_ID=$DOCKER_ID" >> env_vars
+echo "export DOCKER_PASSWD=$DOCKER_PASSWD" >> env_vars
+echo "export KEY_FILENAME=$KEY_FILENAME" >> env_vars
+echo "export OPENSSL_KEY=$OPENSSL_KEY" >> env_vars
+echo "export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> env_vars
+echo "export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> env_vars
+
+# And checkout the correct tag.
+run_on_deploy_box "git fetch"
+run_on_deploy_box "git checkout $CIRCLE_TAG"
+
+# Verify that the tag has been signed by a trusted team member.
+run_on_deploy_box "bash .circleci/verify_tag.sh"
+
+# Copy the necessary environment variables over.
+scp -o StrictHostKeyChecking=no \
+    -i infrastructure/data-refinery-key.pem \
+    -r env_vars ubuntu@$DEPLOY_IP_ADDRESS:refinebio/env_vars
+
+# Decrypt the secrets in our repo.
+run_on_deploy_box "source env_vars && bash .circleci/git_decrypt.sh"
+
+run_on_deploy_box "source env_vars && echo -e '######\nBuilding new images for $CIRCLE_TAG\n######'  &>> /var/log/docker_update.log 2>&1"
+run_on_deploy_box "source env_vars && bash .circleci/update_docker_img.sh >> /var/log/docker_update.log 2>&1"
+run_on_deploy_box "source env_vars && echo -e '######\nFinished building new images for $CIRCLE_TAG\n######'  &>> /var/log/docker_update.log 2>&1"
+
+run_on_deploy_box "source env_vars && echo -e '######\nStarting new deploy for $CIRCLE_TAG\n######' >> /var/log/deploy.log 2>&1"
+run_on_deploy_box "source env_vars && bash .circleci/run_terraform.sh >> /var/log/deploy.log 2>&1"
+run_on_deploy_box "source env_vars && echo -e '######\nDeploying $CIRCLE_TAG finished!\n######' >> /var/log/deploy.log 2>&1"
+
+# Don't leave secrets lying around.
+## Clean out any files we've created or moved so git-crypt will relock the repo.
+run_on_deploy_box "git clean -f"
+run_on_deploy_box "git-crypt lock"

.circleci/run_terraform.sh

@@ -69,7 +69,7 @@ else
 fi
 
 # New deployment
-TF_VAR_user=circleci TF_VAR_stage=$ENVIRONMENT ./deploy.sh -e $ENVIRONMENT -v $CIRCLE_TAG
+./deploy.sh -e $ENVIRONMENT -v $CIRCLE_TAG -u circleci
 exit_code=$?
 
 exit $exit_code

.circleci/update_docker_img.sh

@@ -25,7 +25,7 @@ CCDL_WORKER_IMGS="smasher illumina affymetrix salmon transcriptome no_op downloa
 # required by the workers and data_refinery_foreman images.
 ## Remove old common distributions if they exist
 rm -f ~/refinebio/common/dist/*
-cd ~/refinebio/common && python setup.py sdist
+cd ~/refinebio/common && python3 setup.py sdist
 
 # Log into DockerHub
 docker login -u $DOCKER_ID -p $DOCKER_PASSWD