Merge pull request #763 from LibraryOfCongress/automate-data-load

Automate data load for feature branches

cloudformation/featurebranch.yaml

@@ -17,7 +17,12 @@ Parameters:
         Description: an abbreviation used for creating short-named cloudformation resources
         Type: String
         Default: rel
-
+
+    Priority:
+        Type: Number
+        Description: Priority of the subdomain listener rule, must be unique in the set of listener rules
+        Default: 100
+
 Resources:
 
     RDS:
@@ -30,6 +35,15 @@ Resources:
                 PrivateSubnet1: 'subnet-0aa55b322229b945a'
                 PrivateSubnet2: 'subnet-0f65558b319b2d4dc'
 
+    DataLoadHost:
+        Type: AWS::CloudFormation::Stack
+        Properties:
+            TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/data-load.yaml'
+            Parameters:
+                PostgresqlHost: !GetAtt RDS.Outputs.DatabaseHostName
+                PostgresqlPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword:SecretString:password}}'
+                EnvironmentName: 'test'
+
     ElastiCache:
         Type: AWS::CloudFormation::Stack
         Properties:
@@ -56,3 +70,4 @@ Resources:
                 MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress
                 MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort
                 DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName
+                Priority: !Ref Priority

cloudformation/infrastructure/bastion-hosts.yaml

@@ -6,29 +6,51 @@ Parameters:
     EnvironmentName:
         Description: An environment name that will be prefixed to resource names
         Type: String
-
-    BastionHostsSecurityGroup:
-        Description: The security group for bastion hosts
-        Type: AWS::EC2::SecurityGroup::Id
+        AllowedValues:
+          - dev
+          - test
+          - stage
+          - prod
 
     KeyPairName:
         Description: key pair (within this region) for ECS instances access
         Type: String
 
-    PublicSubnet1:
-        Description: The ID of the public subnet in the first AZ
-        Type: AWS::EC2::Subnet::Id
-
-    PublicSubnet2:
-        Description: The ID of the public subnet in the second AZ
-        Type: AWS::EC2::Subnet::Id
-
 Mappings:
     AWSRegionToAMI:
-        us-east-2:
-            AMI: ami-0cf31d971a3ca20d6
         us-east-1:
-            AMI: ami-04681a1dbd79675a5
+            AMI: ami-0080e4c5bc078760e
+
+    EnvironmentMapping:
+        IamInstanceProfileName:
+            dev: crowd-dev-FargateCluster-WFCY4I0U7JSM-ConcordiaInstanceProfile-RQHLRZADDM9M
+            test: crowd-test-FargateCluster-1R5U1VT4HOYX2-ConcordiaInstanceProfile-1FJXY570ZM2O3
+            stage: crowd-stage-FargateCluster-1TBKSIZQKLJHV-ConcordiaInstanceProfile-1XG3TR3LY42ND
+            prod: crowd-prod-FargateCluster-1X1CI0J3HFJ9F-ConcordiaInstanceProfile-13SHE5FAB7D6Q
+
+        # The ID of the public subnet in the first AZ
+        # Type: AWS::EC2::Subnet::Id
+        PublicSubnet1:
+            dev: subnet-079b5dd4f9acf44e6
+            test: subnet-06f443ea589879e8d
+            stage: subnet-06f40e2fc8d891692
+            prod: subnet-09fdaf1c5c73f588f
+
+        # The ID of the public subnet in the second AZ
+        # Type: AWS::EC2::Subnet::Id
+        PublicSubnet2:
+            dev: subnet-01d6614725c7dabd6
+            test: subnet-05a15c6058ebdf54f
+            stage: subnet-0a022eb0c614b0b00
+            prod: subnet-01580e2a4d6d42b52
+
+        # The security group for bastion hosts
+        # Type: AWS::EC2::SecurityGroup::Id
+        BastionHostsSecurityGroup:
+            dev: sg-062afe8941ace25ad
+            test: sg-0208b0df704b66c3c
+            stage: sg-0a2175a2df32a4332
+            prod: sg-066c68e77787b2a10
 
 Resources:
 
@@ -41,15 +63,35 @@ Resources:
                 - Ref: "AWS::Region"
                 - "AMI"
             InstanceType: "t1.micro"
+            IamInstanceProfile: 
+                Fn::FindInMap: 
+                    - EnvironmentMapping
+                    - IamInstanceProfileName
+                    - Ref: EnvironmentName
             KeyName: 
                 Ref: KeyPairName
             NetworkInterfaces: 
                 - AssociatePublicIpAddress: true
                   DeviceIndex: "0"
                   GroupSet: 
-                    - Ref: BastionHostsSecurityGroup
+                    - Fn::FindInMap: 
+                        - EnvironmentMapping
+                        - BastionHostsSecurityGroup
+                        - Ref: EnvironmentName
                   SubnetId: 
-                    Ref: PublicSubnet1
+                    Fn::FindInMap: 
+                        - EnvironmentMapping
+                        - PublicSubnet1
+                        - Ref: EnvironmentName
+            UserData:
+                Fn::Base64: !Sub |
+                  #!/bin/bash -xe
+                  echo "Running userdata for ${EnvironmentName}"
+                  yum -y update
+                  yum -y install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-6-x86_64/pgdg-ami201503-96-9.6-2.noarch.rpm
+                  yum -y install postgresql96
+                  aws s3 cp s3://crowd-deployment/database-dumps/concordia.latest.dmp concordia.dmp
+
             Tags:
                 - Key: Name
                   Value: !Sub ${EnvironmentName}-BastionHost-1     
@@ -63,15 +105,34 @@ Resources:
                 - Ref: "AWS::Region"
                 - "AMI"
             InstanceType: "t1.micro"
+            IamInstanceProfile: 
+                Fn::FindInMap: 
+                    - EnvironmentMapping
+                    - IamInstanceProfileName
+                    - Ref: EnvironmentName
             KeyName: 
                 Ref: KeyPairName
             NetworkInterfaces: 
                 - AssociatePublicIpAddress: true
                   DeviceIndex: "0"
                   GroupSet: 
-                    - Ref: BastionHostsSecurityGroup
+                    - Fn::FindInMap: 
+                        - EnvironmentMapping
+                        - BastionHostsSecurityGroup
+                        - Ref: EnvironmentName
                   SubnetId: 
-                    Ref: PublicSubnet2
+                    Fn::FindInMap: 
+                        - EnvironmentMapping
+                        - PublicSubnet2
+                        - Ref: EnvironmentName
+            UserData:
+                Fn::Base64: !Sub |
+                  #!/bin/bash -xe
+                  echo "Running userdata for ${EnvironmentName}"
+                  yum -y update
+                  yum -y install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-6-x86_64/pgdg-ami201503-96-9.6-2.noarch.rpm
+                  yum -y install postgresql96
+                  aws s3 cp s3://crowd-deployment/database-dumps/concordia.latest.dmp concordia.dmp
             Tags:
                 - Key: Name
                   Value: !Sub ${EnvironmentName}-BastionHost-2

cloudformation/infrastructure/data-load.yaml

@@ -0,0 +1,95 @@
+Description:
+  This template deploys a host in a private subnet and loads the most recent
+  database dump to the specified database server.
+
+Parameters:
+
+    EnvironmentName:
+        Description: An environment name that will be prefixed to resource names
+        Type: String
+        AllowedValues:
+          - dev
+          - test
+          - stage
+          - prod
+
+    PostgresqlHost:
+        Description: the end point of the RDS database host to restore
+        Type: String
+
+    PostgresqlPassword:
+        Description: the password for the RDS endpoint to restore
+        Type: String
+        NoEcho: true        
+
+Mappings:
+    AWSRegionToAMI:
+        us-east-1:
+            AMI: ami-0080e4c5bc078760e
+
+    EnvironmentMapping:
+        IamInstanceProfileName:
+            dev: crowd-dev-FargateCluster-WFCY4I0U7JSM-ConcordiaInstanceProfile-RQHLRZADDM9M
+            test: crowd-test-FargateCluster-1R5U1VT4HOYX2-ConcordiaInstanceProfile-1FJXY570ZM2O3
+            stage: crowd-stage-FargateCluster-1TBKSIZQKLJHV-ConcordiaInstanceProfile-1XG3TR3LY42ND
+            prod: crowd-prod-FargateCluster-1X1CI0J3HFJ9F-ConcordiaInstanceProfile-13SHE5FAB7D6Q
+
+        PrivateSubnet1:
+            dev: subnet-0c95a830ce007fa65
+            test: subnet-0aa55b322229b945a
+            stage: subnet-0f7c7d66b66d6dd90
+            prod: subnet-0da84976b66c32ce4
+
+        # The security group for bastion hosts
+        # Type: AWS::EC2::SecurityGroup::Id
+        BastionHostsSecurityGroup:
+            dev: sg-062afe8941ace25ad
+            test: sg-0208b0df704b66c3c
+            stage: sg-0a2175a2df32a4332
+            prod: sg-066c68e77787b2a10
+
+Resources:
+
+    DataLoadHost:
+        Type: AWS::EC2::Instance
+        Properties: 
+            ImageId: 
+                Fn::FindInMap: 
+                - AWSRegionToAMI
+                - Ref: "AWS::Region"
+                - "AMI"
+            InstanceType: "t1.micro"
+            IamInstanceProfile: 
+                Fn::FindInMap: 
+                    - EnvironmentMapping
+                    - IamInstanceProfileName
+                    - Ref: EnvironmentName
+            InstanceInitiatedShutdownBehavior: terminate
+            NetworkInterfaces: 
+                - AssociatePublicIpAddress: true
+                  DeviceIndex: "0"
+                  GroupSet: 
+                    - Fn::FindInMap: 
+                        - EnvironmentMapping
+                        - BastionHostsSecurityGroup
+                        - Ref: EnvironmentName
+                  SubnetId: 
+                    Fn::FindInMap: 
+                        - EnvironmentMapping
+                        - PrivateSubnet1
+                        - Ref: EnvironmentName
+            UserData:
+                Fn::Base64: !Sub |
+                  #!/bin/bash -xe
+                  echo "Running userdata for ${EnvironmentName}"
+                  yum -y update
+                  yum -y install https://download.postgresql.org/pub/repos/yum/9.6/redhat/rhel-6-x86_64/pgdg-ami201503-96-9.6-2.noarch.rpm
+                  yum -y install postgresql96
+                  aws s3 cp s3://crowd-deployment/database-dumps/concordia.latest.dmp concordia.dmp
+                  echo "${PostgresqlHost}:5432:*:concordia:${PostgresqlPassword}" >> /root/.pgpass
+                  chmod 0600 /root/.pgpass
+                  pg_restore --create --clean -Fc -U concordia -h ${PostgresqlHost} --dbname=postgres --no-password --no-owner --no-acl concordia.dmp
+                  shutdown -h now
+            Tags:
+                - Key: Name
+                  Value: !Sub ${EnvironmentName}-DataLoadHost

cloudformation/infrastructure/fargate-featurebranch.yaml

@@ -67,6 +67,11 @@ Parameters:
         Type: String
         Description: name of the S3 bucket (public) where exported transcriptions will be stored
 
+    Priority:
+        Type: Number
+        Description: Priority of the subdomain listener rule, must be unique in the set of listener rules
+        Default: 100
+
 
 Resources:
 
@@ -102,7 +107,7 @@ Resources:
           Values:
             - !Ref CanonicalHostName
       ListenerArn: arn:aws:elasticloadbalancing:us-east-1:619333082511:listener/app/crowd-test/81e4820e354ea810/187fd94e534ad833
-      Priority: 100
+      Priority: !Ref Priority
 
   ConcordiaTask:
     Type: AWS::ECS::TaskDefinition

cloudformation/master.yaml

@@ -66,11 +66,6 @@ Parameters:
         Description: which version of the docker images to deploy
         Type: String
         Default: latest
-
-    MyDbPassword:
-        Description: Master database password to use for RDS
-        Type: String
-        NoEcho: true
 
     EnvName:
         Description: which type of environment we are setting up
@@ -143,7 +138,7 @@ Resources:
         Properties:
             TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml"
             Parameters:
-                DbPassword: !Ref MyDbPassword
+                DbPassword: !Sub '{{resolve:secretsmanager:crowd/${EnvName}/DB/MasterUserPassword:SecretString:password}}'
                 DatabaseSecurityGroup: !GetAtt SecurityGroups.Outputs.DatabaseSecurityGroup
                 PrivateSubnet1: !GetAtt VPC.Outputs.PrivateSubnet1
                 PrivateSubnet2: !GetAtt VPC.Outputs.PrivateSubnet2