Cloudformation templates to deploy feature branch to crowd-test

build_containers.sh

@@ -6,6 +6,10 @@ BUILD_ALL=${BUILD_ALL:=0}
 BUILD_NUMBER=${BUILD_NUMBER:=1}
 TAG=${TAG:-test}
 
+BUILD_ALL=${BUILD_ALL:=0}
+BUILD_NUMBER=${BUILD_NUMBER:=1}
+TAG=${TAG:-test}
+
 # Get an unique venv folder to using *inside* workspace
 VENV=".venv-${BUILD_NUMBER}"
 
@@ -29,11 +33,13 @@ eval "$(aws ecr get-login --no-include-email --region us-east-1)"
 python3 setup.py build
 
 docker build -t concordia .
+
 docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}"
 docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}"
 docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}"
 docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}"
 
+
 if [ $BUILD_ALL -eq 1 ]; then
 
     docker build -t concordia/importer --file importer/Dockerfile .
@@ -61,3 +67,4 @@ if [ $BUILD_ALL -eq 1 ]; then
     docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${TAG}"
 
 fi
+

cloudformation/featurebranch.yaml

@@ -0,0 +1,58 @@
+---
+AWSTemplateFormatVersion: '2010-09-09'
+Description: >
+    Deploy a feature branch to a subdomain of crowd-test.loc.gov
+    using pre-existing infrastructure.
+    Assumes docker images have been published to ECR with 
+    tag matching the feature branch name.
+
+Parameters:
+
+    ConcordiaBranch:
+        Description: which branch name to deploy
+        Type: String
+        Default: release
+
+    AbbreviatedName:
+        Description: an abbreviation used for creating short-named cloudformation resources
+        Type: String
+        Default: rel
+
+Resources:
+
+    RDS:
+        Type: AWS::CloudFormation::Stack
+        Properties:
+            TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml'
+            Parameters:
+                DbPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword:SecretString:password}}'
+                DatabaseSecurityGroup: 'sg-0496910b800de2869'
+                PrivateSubnet1: 'subnet-0aa55b322229b945a'
+                PrivateSubnet2: 'subnet-0f65558b319b2d4dc'
+
+    ElastiCache:
+        Type: AWS::CloudFormation::Stack
+        Properties:
+            TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache-feature.yaml'
+            Parameters:
+                EnvironmentName: !Ref AbbreviatedName
+                SecurityGroup: 'sg-028ebfe14211447c4'
+
+    FargateCluster:
+        Type: AWS::CloudFormation::Stack
+        Properties:
+            TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml'
+            Parameters:
+                EnvName: 'test'
+                FullEnvironmentName: 'test'
+                S3BucketName: 'crowd-test-content'
+                ExportS3BucketName: 'crowd-test-export'
+                ConcordiaVersion: !Ref ConcordiaBranch
+                CanonicalHostName: !Sub '${ConcordiaBranch}.crowd-test.loc.gov'
+                VpcId: 'vpc-018e5a73079d0b350'
+                SecurityGroup: 'sg-04de21574623caca7'
+                RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress
+                RedisPort: !GetAtt ElastiCache.Outputs.RedisPort
+                MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress
+                MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort
+                DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName

cloudformation/infrastructure/elasticache-feature.yaml

@@ -0,0 +1,60 @@
+Description: >
+    This template deploys an elasticache cluster to the provided VPC and subnets
+
+Parameters:
+
+    EnvironmentName:
+        Description: An environment name that will be prefixed to resource names
+        Type: String
+
+    SecurityGroup:
+        Description: Select the Security Group to use for the ECS cluster hosts
+        Type: AWS::EC2::SecurityGroup::Id
+
+    CacheNodeType:
+        Type: String
+        Default: cache.m1.small
+
+Resources:
+
+  MemcachedService:
+    Type: AWS::ElastiCache::CacheCluster
+    Properties:
+      VpcSecurityGroupIds:
+        - !Ref 'SecurityGroup'
+      CacheSubnetGroupName: 'crowd-cache-1frtjeewr57u7'
+      CacheNodeType: !Ref 'CacheNodeType'
+      ClusterName: !Sub '${EnvironmentName}-cache'
+      Engine: memcached
+      AutoMinorVersionUpgrade: true
+      NumCacheNodes: 1
+  RedisService:
+    Type: AWS::ElastiCache::CacheCluster
+    Properties:
+      VpcSecurityGroupIds:
+        - !Ref 'SecurityGroup'
+      CacheSubnetGroupName: 'crowd-cache-1frtjeewr57u7'
+      CacheNodeType: !Ref 'CacheNodeType'
+      ClusterName: !Sub '${EnvironmentName}-redis'
+      Engine: redis
+      AutoMinorVersionUpgrade: true
+      NumCacheNodes: 1
+      SnapshotRetentionLimit: 1
+
+Outputs:
+
+  RedisAddress:
+    Description: Redis endpoint address
+    Value: !GetAtt 'RedisService.RedisEndpoint.Address'
+
+  RedisPort:
+    Description: Redis endpoint port
+    Value: !GetAtt 'RedisService.RedisEndpoint.Port'
+
+  MemcachedAddress:
+    Description: memcached endpoint address
+    Value: !GetAtt 'MemcachedService.ConfigurationEndpoint.Address'
+
+  MemcachedPort:
+    Description: memcached endpoint port
+    Value: !GetAtt 'MemcachedService.ConfigurationEndpoint.Port'

cloudformation/infrastructure/fargate-cluster.yaml

@@ -219,7 +219,7 @@ Resources:
   ConcordiaTask:
     Type: AWS::ECS::TaskDefinition
     Properties:
-      Family: !Sub concordia-${EnvName}
+      Family: !Sub crowd-${EnvName}
       Cpu: '4096'
       Memory: '16384'
       NetworkMode: awsvpc

cloudformation/infrastructure/fargate-featurebranch.yaml

@@ -0,0 +1,190 @@
+Description: >
+    This template deploys a fargate cluster to the provided VPC and subnets
+
+Parameters:
+
+    SecurityGroup:
+        Description: Select the Security Group to use for the ECS cluster hosts
+        Type: AWS::EC2::SecurityGroup::Id
+
+    VpcId:
+        Description: The Id of the VPC for this cluster
+        Type: AWS::EC2::VPC::Id
+
+    ConcordiaVersion:
+        Type: String
+        Description: docker tag of concordia app image to pull and deploy
+        Default: latest
+
+    EnvName:
+        Type: String
+        Description: which environment to target
+        AllowedValues:
+            - 'dev'
+            - 'test'
+            - 'stage'
+            - 'prod'
+        ConstraintDescription: Must match a location for secret storage in secretsmanager
+
+    FullEnvironmentName:
+        Type: String
+        Description: Full name of deployment environment
+        AllowedValues:
+            - 'development'
+            - 'test'
+            - 'staging'
+            - 'production'
+
+    RedisAddress:
+        Type: String
+        Description: Redis endpoint address
+
+    RedisPort:
+        Type: String
+        Description: Redis endpoint port
+
+    MemcachedAddress:
+        Type: String
+        Description: memcached endpoint address
+
+    MemcachedPort:
+        Type: String
+        Description: memcached endpoint port
+
+    CanonicalHostName:
+        Type: String
+        Description: canonical host name of the application, e.g. crowd-test.loc.gov
+
+    DatabaseEndpoint:
+        Type: String
+        Description: Host name of the Postgres RDS service
+
+    S3BucketName:
+        Type: String
+        Description: name of the S3 bucket (public) where collection images will be stored
+
+    ExportS3BucketName:
+        Type: String
+        Description: name of the S3 bucket (public) where exported transcriptions will be stored
+
+
+Resources:
+
+  ConcordiaAppLogsGroup:
+    Type: AWS::Logs::LogGroup
+    Properties:
+      LogGroupName: !Ref AWS::StackName
+      RetentionInDays: 30
+
+
+  ConcordiaExternalTargetGroup:
+    Type: AWS::ElasticLoadBalancingV2::TargetGroup
+    Properties:
+      HealthCheckIntervalSeconds: 30
+      HealthCheckPath: /healthz
+      HealthCheckProtocol: HTTP
+      HealthCheckTimeoutSeconds: 5
+      HealthyThresholdCount: 2
+      UnhealthyThresholdCount: 10
+      TargetType: ip
+      Port: 80
+      Protocol: HTTP
+      VpcId: !Ref VpcId
+
+  SubdomainListenerRule:
+    Type: AWS::ElasticLoadBalancingV2::ListenerRule
+    Properties:
+      Actions:
+        - TargetGroupArn: !Ref ConcordiaExternalTargetGroup
+          Type: forward
+      Conditions:
+        - Field: host-header
+          Values:
+            - !Ref CanonicalHostName
+      ListenerArn: arn:aws:elasticloadbalancing:us-east-1:619333082511:listener/app/crowd-test/81e4820e354ea810/187fd94e534ad833
+      Priority: 100
+
+  ConcordiaTask:
+    Type: AWS::ECS::TaskDefinition
+    Properties:
+      Family: !Sub crowd-${ConcordiaVersion}
+      Cpu: '2048'
+      Memory: '8192'
+      NetworkMode: awsvpc
+      RequiresCompatibilities:
+        - FARGATE
+      ExecutionRoleArn: ecsTaskExecutionRole
+      TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/ConcordiaServerTaskRole-crowd-test'
+      Volumes:
+        - Name: images_volume
+      ContainerDefinitions:
+        - Name: app
+          Cpu: 2048
+          Memory: 8192
+          Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia:${ConcordiaVersion}'
+          LogConfiguration:
+            LogDriver: awslogs
+            Options:
+              awslogs-group: !Ref 'ConcordiaAppLogsGroup'
+              awslogs-region: !Ref 'AWS::Region'
+              awslogs-stream-prefix: ConcordiaServer
+          Environment:
+              - Name: AWS
+                Value: '1'
+              - Name: ENV_NAME
+                Value: !Ref EnvName
+              - Name: CONCORDIA_ENVIRONMENT
+                Value: !Ref FullEnvironmentName
+              - Name: S3_BUCKET_NAME
+                Value: !Ref S3BucketName
+              - Name: EXPORT_S3_BUCKET_NAME
+                Value: !Ref ExportS3BucketName
+              - Name: CELERY_BROKER_URL
+                Value: pyamqp://guest@localhost:5672
+              - Name: AWS_DEFAULT_REGION
+                Value: !Ref AWS::Region
+              - Name: SENTRY_BACKEND_DSN
+                Value: http://[email protected]/2
+              - Name: SENTRY_FRONTEND_DSN
+                Value: https://[email protected]/3
+              - Name: REDIS_ADDRESS
+                Value: !Ref RedisAddress
+              - Name: REDIS_PORT
+                Value: !Ref RedisPort
+              - Name: MEMCACHED_ADDRESS
+                Value: !Ref MemcachedAddress
+              - Name: MEMCACHED_PORT
+                Value: !Ref MemcachedPort
+              - Name: POSTGRESQL_HOST
+                Value: !Ref DatabaseEndpoint
+              - Name: HOST_NAME
+                Value: !Ref CanonicalHostName
+              - Name: DJANGO_SETTINGS_MODULE
+                Value: concordia.settings_ecs  
+          MountPoints:
+              - SourceVolume: images_volume
+                ContainerPath: /concordia_images
+          PortMappings:
+              - ContainerPort: 80
+
+  ConcordiaExternalService:
+    Type: AWS::ECS::Service
+    Properties:
+      Cluster: crowd-test
+      LaunchType: FARGATE
+      DeploymentConfiguration:
+        MaximumPercent: 200
+        MinimumHealthyPercent: 75
+      DesiredCount: 1
+      NetworkConfiguration:
+        AwsvpcConfiguration:
+          SecurityGroups:
+            - !Ref SecurityGroup
+          Subnets:
+            - subnet-0aa55b322229b945a
+            - subnet-0f65558b319b2d4dc
+      TaskDefinition: !Ref ConcordiaTask
+      LoadBalancers:
+        - ContainerName: 'app'
+          ContainerPort: 80
+          TargetGroupArn: !Ref ConcordiaExternalTargetGroup

cloudformation/master.yaml

@@ -7,10 +7,10 @@ Description: >
     route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), 
     and default routes for them in the private subnets.
 
-    It then deploys a highly available ECS cluster using an AutoScaling Group, with 
-    ECS hosts distributed across multiple Availability Zones. 
+    It then deploys a Fargate ECS cluster distributed across multiple
+    Availability Zones. 
 
-    Finally, it deploys a pair of example ECS services from containers published in 
+    Finally, it deploys crowd ECS services from containers published in 
     Amazon EC2 Container Registry (Amazon ECR).
 Mappings:
     EnvironmentMapping: