From 6aa3bbd36a2933e2dbec0feb0f576caeb5a77faa Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Thu, 10 Jan 2019 13:36:35 -0500 Subject: [PATCH 01/18] If Django secret key is not in the env, use random key in settings_ecs --- concordia/settings_ecs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/concordia/settings_ecs.py b/concordia/settings_ecs.py index 5230191fb..1b249abc8 100644 --- a/concordia/settings_ecs.py +++ b/concordia/settings_ecs.py @@ -1,6 +1,8 @@ import json import os +from django.core.management import get_random_secret_key + from .secrets import get_secret from .settings_template import * # NOQA ignore=F405 from .settings_template import CONCORDIA_ENVIRONMENT, DATABASES, INSTALLED_APPS, LOGGING @@ -32,7 +34,7 @@ EMAIL_HOST_PASSWORD = smtp_secret["Password"] else: - DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "changeme") + DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", get_random_secret_key()) EMAIL_HOST = os.environ.get("EMAIL_HOST", "localhost") EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER", "") EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD", "") From d2cdf9b8967c17482df2fa48c22ddac1fabe738b Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Thu, 10 Jan 2019 13:52:59 -0500 Subject: [PATCH 02/18] first pass at cloudformation template for deploying feature branches to test --- cloudformation/feature.yaml | 85 +++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 cloudformation/feature.yaml diff --git a/cloudformation/feature.yaml b/cloudformation/feature.yaml new file mode 100644 index 000000000..b0522b69d --- /dev/null +++ b/cloudformation/feature.yaml @@ -0,0 +1,85 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: > + + This template deploys a VPC, with a pair of public and private subnets spread + across two Availabilty Zones. It deploys an Internet Gateway, with a default + route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), + and default routes for them in the private subnets. + + It then deploys a highly available ECS cluster using an AutoScaling Group, with + ECS hosts distributed across multiple Availability Zones. + + Finally, it deploys a pair of example ECS services from containers published in + Amazon EC2 Container Registry (Amazon ECR). + + +Parameters: + + ConcordiaBranch: + Description: which branch name / docker tagged images to deploy + Type: String + Default: release + + CanonicalHostName: + Description: the canonical host name for this environment + Type: String + AllowedValues: + - "crowd-dev.loc.gov" + - "crowd-test.loc.gov" + - "crowd-stage.loc.gov" + - "crowd-prod.loc.gov" + - "crowd.loc.gov" + +Resources: + + RDS: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml" + Parameters: + DbPassword: !Ref MyDbPassword + DatabaseSecurityGroup: !GetAtt SecurityGroups.Outputs.DatabaseSecurityGroup + PrivateSubnet1: !GetAtt VPC.Outputs.PrivateSubnet1 + PrivateSubnet2: !GetAtt VPC.Outputs.PrivateSubnet2 + + ElastiCache: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache.yaml" + Parameters: + EnvironmentName: !Ref AWS::StackName + SecurityGroup: !GetAtt SecurityGroups.Outputs.CacheServiceSecurityGroup + PrivateSubnets: !GetAtt VPC.Outputs.PrivateSubnets + + FargateCluster: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-cluster.yaml" + Parameters: + EnvironmentName: !Ref AWS::StackName + EnvName: "test" + FullEnvironmentName: "test" + S3BucketName: "crowd-test-content" + ExportS3BucketName: "crowd-test-export" + ConcordiaVersion: !Ref ConcordiaBranch + CanonicalHostName: !Ref CanonicalHostName + + VpcId: !GetAtt VPC.Outputs.VPC + SecurityGroup: !GetAtt SecurityGroups.Outputs.ECSHostSecurityGroup + LoadBalancerSecurityGroup: !GetAtt SecurityGroups.Outputs.LoadBalancerSecurityGroup + PrivateSubnets: !GetAtt VPC.Outputs.PrivateSubnets + PublicSubnets: !GetAtt VPC.Outputs.PublicSubnets + + RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress + RedisPort: !GetAtt ElastiCache.Outputs.RedisPort + MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress + MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort + DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName + + +Outputs: + + WebsiteServiceUrl: + Description: The URL endpoint for the concordia website service + Value: !Join ["", [ !GetAtt FargateCluster.Outputs.LoadBalancerUrl, "/" ]] From 0d4d166677d486f21db32684267af592c516110e Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Fri, 11 Jan 2019 11:57:50 -0500 Subject: [PATCH 03/18] WIP deploying feature branches to crowd-test --- cloudformation/feature.yaml | 85 ----- cloudformation/featurebranch.yaml | 65 ++++ .../infrastructure/fargate-cluster.yaml | 6 +- .../infrastructure/fargate-featurebranch.yaml | 309 ++++++++++++++++++ cloudformation/master.yaml | 7 +- 5 files changed, 383 insertions(+), 89 deletions(-) delete mode 100644 cloudformation/feature.yaml create mode 100644 cloudformation/featurebranch.yaml create mode 100644 cloudformation/infrastructure/fargate-featurebranch.yaml diff --git a/cloudformation/feature.yaml b/cloudformation/feature.yaml deleted file mode 100644 index b0522b69d..000000000 --- a/cloudformation/feature.yaml +++ /dev/null @@ -1,85 +0,0 @@ ---- -AWSTemplateFormatVersion: '2010-09-09' -Description: > - - This template deploys a VPC, with a pair of public and private subnets spread - across two Availabilty Zones. It deploys an Internet Gateway, with a default - route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), - and default routes for them in the private subnets. - - It then deploys a highly available ECS cluster using an AutoScaling Group, with - ECS hosts distributed across multiple Availability Zones. - - Finally, it deploys a pair of example ECS services from containers published in - Amazon EC2 Container Registry (Amazon ECR). - - -Parameters: - - ConcordiaBranch: - Description: which branch name / docker tagged images to deploy - Type: String - Default: release - - CanonicalHostName: - Description: the canonical host name for this environment - Type: String - AllowedValues: - - "crowd-dev.loc.gov" - - "crowd-test.loc.gov" - - "crowd-stage.loc.gov" - - "crowd-prod.loc.gov" - - "crowd.loc.gov" - -Resources: - - RDS: - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml" - Parameters: - DbPassword: !Ref MyDbPassword - DatabaseSecurityGroup: !GetAtt SecurityGroups.Outputs.DatabaseSecurityGroup - PrivateSubnet1: !GetAtt VPC.Outputs.PrivateSubnet1 - PrivateSubnet2: !GetAtt VPC.Outputs.PrivateSubnet2 - - ElastiCache: - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache.yaml" - Parameters: - EnvironmentName: !Ref AWS::StackName - SecurityGroup: !GetAtt SecurityGroups.Outputs.CacheServiceSecurityGroup - PrivateSubnets: !GetAtt VPC.Outputs.PrivateSubnets - - FargateCluster: - Type: AWS::CloudFormation::Stack - Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-cluster.yaml" - Parameters: - EnvironmentName: !Ref AWS::StackName - EnvName: "test" - FullEnvironmentName: "test" - S3BucketName: "crowd-test-content" - ExportS3BucketName: "crowd-test-export" - ConcordiaVersion: !Ref ConcordiaBranch - CanonicalHostName: !Ref CanonicalHostName - - VpcId: !GetAtt VPC.Outputs.VPC - SecurityGroup: !GetAtt SecurityGroups.Outputs.ECSHostSecurityGroup - LoadBalancerSecurityGroup: !GetAtt SecurityGroups.Outputs.LoadBalancerSecurityGroup - PrivateSubnets: !GetAtt VPC.Outputs.PrivateSubnets - PublicSubnets: !GetAtt VPC.Outputs.PublicSubnets - - RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress - RedisPort: !GetAtt ElastiCache.Outputs.RedisPort - MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress - MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort - DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName - - -Outputs: - - WebsiteServiceUrl: - Description: The URL endpoint for the concordia website service - Value: !Join ["", [ !GetAtt FargateCluster.Outputs.LoadBalancerUrl, "/" ]] diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml new file mode 100644 index 000000000..1c277a798 --- /dev/null +++ b/cloudformation/featurebranch.yaml @@ -0,0 +1,65 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: > + Deploy a feature branch to a subdomain of crowd-test.loc.gov + using pre-existing infrastructure + Assumes docker images have been published to ECR with + tag matching the feature branch name. + +Parameters: + + ConcordiaBranch: + Description: which branch name to deploy + Type: String + Default: release + +Resources: + + RDS: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml" + Parameters: + DbPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword-rxyUBT:SecretString:password}}' + DatabaseSecurityGroup: sg-0496910b800de2869 + PrivateSubnet1: subnet-0aa55b322229b945a + PrivateSubnet2: subnet-0f65558b319b2d4dc + + ElastiCache: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache.yaml" + Parameters: + EnvironmentName: !Ref AWS::StackName + SecurityGroup: sg-028ebfe14211447c4 + PrivateSubnets: + - subnet-0aa55b322229b945a + - subnet-0f65558b319b2d4dc + + FargateCluster: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml" + Parameters: + EnvironmentName: !Ref AWS::StackName + EnvName: test + FullEnvironmentName: test + S3BucketName: crowd-test-content + ExportS3BucketName: crowd-test-export + ConcordiaVersion: !Ref ConcordiaBranch + CanonicalHostName: !Sub ${ConcordiaBranch}.crowd-test.loc.gov + CertificateName: crowd-test.loc.gov + + VpcId: !GetAtt VPC.Outputs.VPC + SecurityGroup: !GetAtt SecurityGroups.Outputs.ECSHostSecurityGroup + LoadBalancerSecurityGroup: !GetAtt SecurityGroups.Outputs.LoadBalancerSecurityGroup + PrivateSubnets: + - subnet-0aa55b322229b945a + - subnet-0f65558b319b2d4dc + PublicSubnets: !GetAtt VPC.Outputs.PublicSubnets + + RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress + RedisPort: !GetAtt ElastiCache.Outputs.RedisPort + MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress + MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort + DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName diff --git a/cloudformation/infrastructure/fargate-cluster.yaml b/cloudformation/infrastructure/fargate-cluster.yaml index e61e06667..01cb5611b 100644 --- a/cloudformation/infrastructure/fargate-cluster.yaml +++ b/cloudformation/infrastructure/fargate-cluster.yaml @@ -71,6 +71,10 @@ Parameters: Type: String Description: canonical host name of the application, e.g. crowd-test.loc.gov + CertificateName: + Type: String + Description: Normally the canonical host name, except for subdomains in crowd-test.loc.gov + DatabaseEndpoint: Type: String Description: Host name of the Postgres RDS service @@ -202,7 +206,7 @@ Resources: SecureExternalLoadBalancerListener: Properties: Certificates: - - CertificateArn: !Sub 'arn:aws:iam::${AWS::AccountId}:server-certificate/${CanonicalHostName}' + - CertificateArn: !Sub 'arn:aws:iam::${AWS::AccountId}:server-certificate/${CertificateName}' DefaultActions: - TargetGroupArn: !Ref ConcordiaExternalTargetGroup Type: forward diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml new file mode 100644 index 000000000..4fa06df8e --- /dev/null +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -0,0 +1,309 @@ +Description: > + This template deploys a fargate cluster to the provided VPC and subnets + +Parameters: + + EnvironmentName: + Description: An environment name that will be prefixed to resource names + Type: String + + PublicSubnets: + Description: The subnets for the load balancer + Type: List + + PrivateSubnets: + Description: Choose which subnets this ECS cluster should be deployed to + Type: List + + SecurityGroup: + Description: Select the Security Group to use for the ECS cluster hosts + Type: AWS::EC2::SecurityGroup::Id + + LoadBalancerSecurityGroup: + Description: The SecurityGroup for load balancer + Type: AWS::EC2::SecurityGroup::Id + + VpcId: + Description: The Id of the VPC for this cluster + Type: AWS::EC2::VPC::Id + + ConcordiaVersion: + Type: String + Description: version of concordia, concordia/importer, and rabbitmq docker images to pull and deploy + Default: latest + + EnvName: + Type: String + Description: which environment to target + AllowedValues: + - 'dev' + - 'test' + - 'stage' + - 'prod' + ConstraintDescription: Must match a location for secret storage in secretsmanager + + FullEnvironmentName: + Type: String + Description: Full name of deployment environment + AllowedValues: + - 'development' + - 'test' + - 'staging' + - 'production' + + RedisAddress: + Type: String + Description: Redis endpoint address + + RedisPort: + Type: String + Description: Redis endpoint port + + MemcachedAddress: + Type: String + Description: memcached endpoint address + + MemcachedPort: + Type: String + Description: memcached endpoint port + + CanonicalHostName: + Type: String + Description: canonical host name of the application, e.g. crowd-test.loc.gov + + CertificateName: + Type: String + Description: Normally the canonical host name, except for subdomains in crowd-test.loc.gov + + DatabaseEndpoint: + Type: String + Description: Host name of the Postgres RDS service + + S3BucketName: + Type: String + Description: name of the S3 bucket (public) where collection images will be stored + + ExportS3BucketName: + Type: String + Description: name of the S3 bucket (public) where exported transcriptions will be stored + + +Resources: + + ConcordiaExternalTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + HealthCheckIntervalSeconds: 30 + HealthCheckPath: /healthz + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 5 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 10 + TargetType: ip + Port: 80 + Protocol: HTTP + VpcId: !Ref VpcId + +SubdomainListenerRule: + Type: AWS::ElasticLoadBalancingV2::ListenerRule + Properties: + Actions: + - TargetGroupArn: !Ref ConcordiaExternalTargetGroup + Type: forward + Conditions: + - Field: host-header + Values: + - !Ref CanonicalHostName + ListenerArn: arn:aws:elasticloadbalancing:us-east-1:619333082511:listener/app/crowd-test/81e4820e354ea810/187fd94e534ad833 + Priority: 100 + + ConcordiaTask: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub concordia-${EnvName} + Cpu: '4096' + Memory: '16384' + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: ecsTaskExecutionRole + TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/ConcordiaServerTaskRole-${EnvironmentName}' + Volumes: + - Name: images_volume + ContainerDefinitions: + - Name: app + Cpu: 2048 + Memory: 8192 + Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia:${ConcordiaVersion}' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref 'ConcordiaAppLogsGroup' + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ConcordiaServer + Environment: + - Name: AWS + Value: '1' + - Name: ENV_NAME + Value: !Ref EnvName + - Name: CONCORDIA_ENVIRONMENT + Value: !Ref FullEnvironmentName + - Name: S3_BUCKET_NAME + Value: !Ref S3BucketName + - Name: EXPORT_S3_BUCKET_NAME + Value: !Ref ExportS3BucketName + - Name: CELERY_BROKER_URL + Value: pyamqp://guest@localhost:5672 + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: SENTRY_BACKEND_DSN + Value: http://34db819263f34c28809da045f841f045@sentry-internal.devops.cloud/2 + - Name: SENTRY_FRONTEND_DSN + Value: https://48ec47f15e484502a29879e40ed2e0c3@crowd-sentry.loc.gov/3 + - Name: REDIS_ADDRESS + Value: !Ref RedisAddress + - Name: REDIS_PORT + Value: !Ref RedisPort + - Name: MEMCACHED_ADDRESS + Value: !Ref MemcachedAddress + - Name: MEMCACHED_PORT + Value: !Ref MemcachedPort + - Name: POSTGRESQL_HOST + Value: !Ref DatabaseEndpoint + - Name: HOST_NAME + Value: !Ref CanonicalHostName + - Name: DJANGO_SETTINGS_MODULE + Value: concordia.settings_ecs + MountPoints: + - SourceVolume: images_volume + ContainerPath: /concordia_images + PortMappings: + - ContainerPort: 80 + - Name: rabbit + Cpu: 1024 + Memory: 2048 + Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/rabbitmq:${ConcordiaVersion}' + PortMappings: + - ContainerPort: 5672 + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref 'ConcordiaAppLogsGroup' + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ConcordiaCron + - Name: importer + Cpu: 1024 + Memory: 2048 + Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia/importer:${ConcordiaVersion}' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref 'ConcordiaAppLogsGroup' + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ConcordiaWorker + Environment: + - Name: AWS + Value: '1' + - Name: ENV_NAME + Value: !Ref EnvName + - Name: CONCORDIA_ENVIRONMENT + Value: !Ref FullEnvironmentName + - Name: S3_BUCKET_NAME + Value: !Ref S3BucketName + - Name: EXPORT_S3_BUCKET_NAME + Value: !Ref ExportS3BucketName + - Name: CELERY_BROKER_URL + Value: pyamqp://guest@localhost:5672 + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: SENTRY_BACKEND_DSN + Value: http://34db819263f34c28809da045f841f045@sentry-internal.devops.cloud/2 + - Name: SENTRY_FRONTEND_DSN + Value: https://48ec47f15e484502a29879e40ed2e0c3@crowd-sentry.loc.gov/3 + - Name: REDIS_ADDRESS + Value: !Ref RedisAddress + - Name: REDIS_PORT + Value: !Ref RedisPort + - Name: MEMCACHED_ADDRESS + Value: !Ref MemcachedAddress + - Name: MEMCACHED_PORT + Value: !Ref MemcachedPort + - Name: POSTGRESQL_HOST + Value: !Ref DatabaseEndpoint + - Name: HOST_NAME + Value: !Ref CanonicalHostName + - Name: DJANGO_SETTINGS_MODULE + Value: concordia.settings_ecs + MountPoints: + - SourceVolume: images_volume + ContainerPath: /concordia_images + - Name: celerybeat + Cpu: 1024 + Memory: 2048 + Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia/celerybeat:${ConcordiaVersion}' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref 'ConcordiaAppLogsGroup' + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ConcordiaWorker + Environment: + - Name: AWS + Value: '1' + - Name: ENV_NAME + Value: !Ref EnvName + - Name: CONCORDIA_ENVIRONMENT + Value: !Ref FullEnvironmentName + - Name: S3_BUCKET_NAME + Value: !Ref S3BucketName + - Name: EXPORT_S3_BUCKET_NAME + Value: !Ref ExportS3BucketName + - Name: CELERY_BROKER_URL + Value: pyamqp://guest@localhost:5672 + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: SENTRY_BACKEND_DSN + Value: http://34db819263f34c28809da045f841f045@sentry-internal.devops.cloud/2 + - Name: SENTRY_FRONTEND_DSN + Value: https://48ec47f15e484502a29879e40ed2e0c3@crowd-sentry.loc.gov/3 + - Name: REDIS_ADDRESS + Value: !Ref RedisAddress + - Name: REDIS_PORT + Value: !Ref RedisPort + - Name: MEMCACHED_ADDRESS + Value: !Ref MemcachedAddress + - Name: MEMCACHED_PORT + Value: !Ref MemcachedPort + - Name: POSTGRESQL_HOST + Value: !Ref DatabaseEndpoint + - Name: HOST_NAME + Value: !Ref CanonicalHostName + - Name: DJANGO_SETTINGS_MODULE + Value: concordia.settings_ecs + + ConcordiaExternalService: + Type: AWS::ECS::Service + DependsOn: ExternalLoadBalancerListener + Properties: + Cluster: !Ref ECSCluster + LaunchType: FARGATE + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 75 + DesiredCount: 1 + NetworkConfiguration: + AwsvpcConfiguration: + SecurityGroups: + - !Ref SecurityGroup + Subnets: !Ref PrivateSubnets + TaskDefinition: !Ref ConcordiaTask + LoadBalancers: + - ContainerName: 'app' + ContainerPort: 80 + TargetGroupArn: !Ref ConcordiaExternalTargetGroup + +Outputs: + + LoadBalancerUrl: + Description: The URL of the ALB + Value: !GetAtt LoadBalancer.DNSName \ No newline at end of file diff --git a/cloudformation/master.yaml b/cloudformation/master.yaml index 8ff92b192..f4a73058e 100644 --- a/cloudformation/master.yaml +++ b/cloudformation/master.yaml @@ -7,10 +7,10 @@ Description: > route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), and default routes for them in the private subnets. - It then deploys a highly available ECS cluster using an AutoScaling Group, with - ECS hosts distributed across multiple Availability Zones. + It then deploys a Fargate ECS cluster distributed across multiple + Availability Zones. - Finally, it deploys a pair of example ECS services from containers published in + Finally, it deploys crowd ECS services from containers published in Amazon EC2 Container Registry (Amazon ECR). Mappings: EnvironmentMapping: @@ -175,6 +175,7 @@ Resources: MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort CanonicalHostName: !Ref CanonicalHostName + CertificateName: !Ref CanonicalHostName DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName FullEnvironmentName: !Ref FullEnvironmentName S3BucketName: !FindInMap [EnvironmentMapping, S3BucketNameMap, !Ref EnvName] From 7e1c94aa19c7b41000dacd488954a6161cd1045d Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 11:55:40 -0500 Subject: [PATCH 04/18] Remove unused deployment code --- cloudformation/setup.sh | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 cloudformation/setup.sh diff --git a/cloudformation/setup.sh b/cloudformation/setup.sh deleted file mode 100644 index 2c25ab12d..000000000 --- a/cloudformation/setup.sh +++ /dev/null @@ -1,14 +0,0 @@ -# export AWS_ACCESS_KEY="" -# export AWS_SECRET_KEY="" -# export AWS_CREDENTIAL_FILE="" - -# Make sure your aws credentials are configured before attempting to run this script. - -S3_BUCKET_NAME = "rstorey-concordia-refarch" -STACK_NAME = "rstorey-refarch-test" - -aws s3api create-bucket --bucket $S3_BUCKET_NAME --region us-east-1 - -aws s3 sync . s3://$S3_BUCKET_NAME - -aws cloudformation create-stack --stack-name $STACK_NAME --template-url https://s3.amazonaws.com/$S3_BUCKET_NAME/master.yaml \ No newline at end of file From 8cd1d8bdc3545572c58483683cdecdeb9725f85c Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 11:56:48 -0500 Subject: [PATCH 05/18] populate values for deploying feature branches to existing infrastructure --- cloudformation/featurebranch.yaml | 11 +--- .../infrastructure/fargate-cluster.yaml | 2 +- .../infrastructure/fargate-featurebranch.yaml | 56 ++++++++----------- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml index 1c277a798..abc924f27 100644 --- a/cloudformation/featurebranch.yaml +++ b/cloudformation/featurebranch.yaml @@ -32,7 +32,7 @@ Resources: Parameters: EnvironmentName: !Ref AWS::StackName SecurityGroup: sg-028ebfe14211447c4 - PrivateSubnets: + PrivateSubnets: - subnet-0aa55b322229b945a - subnet-0f65558b319b2d4dc @@ -48,16 +48,11 @@ Resources: ExportS3BucketName: crowd-test-export ConcordiaVersion: !Ref ConcordiaBranch CanonicalHostName: !Sub ${ConcordiaBranch}.crowd-test.loc.gov - CertificateName: crowd-test.loc.gov - - VpcId: !GetAtt VPC.Outputs.VPC - SecurityGroup: !GetAtt SecurityGroups.Outputs.ECSHostSecurityGroup - LoadBalancerSecurityGroup: !GetAtt SecurityGroups.Outputs.LoadBalancerSecurityGroup + VpcId: vpc-018e5a73079d0b350 + SecurityGroup: sg-04de21574623caca7 PrivateSubnets: - subnet-0aa55b322229b945a - subnet-0f65558b319b2d4dc - PublicSubnets: !GetAtt VPC.Outputs.PublicSubnets - RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress RedisPort: !GetAtt ElastiCache.Outputs.RedisPort MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress diff --git a/cloudformation/infrastructure/fargate-cluster.yaml b/cloudformation/infrastructure/fargate-cluster.yaml index 01cb5611b..49b2748a2 100644 --- a/cloudformation/infrastructure/fargate-cluster.yaml +++ b/cloudformation/infrastructure/fargate-cluster.yaml @@ -223,7 +223,7 @@ Resources: ConcordiaTask: Type: AWS::ECS::TaskDefinition Properties: - Family: !Sub concordia-${EnvName} + Family: !Sub crowd-${EnvName} Cpu: '4096' Memory: '16384' NetworkMode: awsvpc diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml index 4fa06df8e..5b31f9c83 100644 --- a/cloudformation/infrastructure/fargate-featurebranch.yaml +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -7,10 +7,6 @@ Parameters: Description: An environment name that will be prefixed to resource names Type: String - PublicSubnets: - Description: The subnets for the load balancer - Type: List - PrivateSubnets: Description: Choose which subnets this ECS cluster should be deployed to Type: List @@ -19,10 +15,6 @@ Parameters: Description: Select the Security Group to use for the ECS cluster hosts Type: AWS::EC2::SecurityGroup::Id - LoadBalancerSecurityGroup: - Description: The SecurityGroup for load balancer - Type: AWS::EC2::SecurityGroup::Id - VpcId: Description: The Id of the VPC for this cluster Type: AWS::EC2::VPC::Id @@ -71,10 +63,6 @@ Parameters: Type: String Description: canonical host name of the application, e.g. crowd-test.loc.gov - CertificateName: - Type: String - Description: Normally the canonical host name, except for subdomains in crowd-test.loc.gov - DatabaseEndpoint: Type: String Description: Host name of the Postgres RDS service @@ -90,6 +78,13 @@ Parameters: Resources: + ConcordiaAppLogsGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Ref AWS::StackName + RetentionInDays: 30 + + ConcordiaExternalTargetGroup: Type: AWS::ElasticLoadBalancingV2::TargetGroup Properties: @@ -104,23 +99,23 @@ Resources: Protocol: HTTP VpcId: !Ref VpcId -SubdomainListenerRule: - Type: AWS::ElasticLoadBalancingV2::ListenerRule - Properties: - Actions: - - TargetGroupArn: !Ref ConcordiaExternalTargetGroup - Type: forward - Conditions: - - Field: host-header - Values: - - !Ref CanonicalHostName - ListenerArn: arn:aws:elasticloadbalancing:us-east-1:619333082511:listener/app/crowd-test/81e4820e354ea810/187fd94e534ad833 - Priority: 100 + SubdomainListenerRule: + Type: AWS::ElasticLoadBalancingV2::ListenerRule + Properties: + Actions: + - TargetGroupArn: !Ref ConcordiaExternalTargetGroup + Type: forward + Conditions: + - Field: host-header + Values: + - !Ref CanonicalHostName + ListenerArn: arn:aws:elasticloadbalancing:us-east-1:619333082511:listener/app/crowd-test/81e4820e354ea810/187fd94e534ad833 + Priority: 100 ConcordiaTask: Type: AWS::ECS::TaskDefinition Properties: - Family: !Sub concordia-${EnvName} + Family: !Sub crowd-${EnvName} Cpu: '4096' Memory: '16384' NetworkMode: awsvpc @@ -283,9 +278,8 @@ SubdomainListenerRule: ConcordiaExternalService: Type: AWS::ECS::Service - DependsOn: ExternalLoadBalancerListener Properties: - Cluster: !Ref ECSCluster + Cluster: crowd-test LaunchType: FARGATE DeploymentConfiguration: MaximumPercent: 200 @@ -300,10 +294,4 @@ SubdomainListenerRule: LoadBalancers: - ContainerName: 'app' ContainerPort: 80 - TargetGroupArn: !Ref ConcordiaExternalTargetGroup - -Outputs: - - LoadBalancerUrl: - Description: The URL of the ALB - Value: !GetAtt LoadBalancer.DNSName \ No newline at end of file + TargetGroupArn: !Ref ConcordiaExternalTargetGroup \ No newline at end of file From a1402adeb71f3780bbda88d878c39a113474aea2 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 12:05:33 -0500 Subject: [PATCH 06/18] Move private subnets to avoid cfn-lint warning, delete unneeded containers --- cloudformation/featurebranch.yaml | 3 - .../infrastructure/fargate-featurebranch.yaml | 109 +----------------- 2 files changed, 3 insertions(+), 109 deletions(-) diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml index abc924f27..95ea00f61 100644 --- a/cloudformation/featurebranch.yaml +++ b/cloudformation/featurebranch.yaml @@ -50,9 +50,6 @@ Resources: CanonicalHostName: !Sub ${ConcordiaBranch}.crowd-test.loc.gov VpcId: vpc-018e5a73079d0b350 SecurityGroup: sg-04de21574623caca7 - PrivateSubnets: - - subnet-0aa55b322229b945a - - subnet-0f65558b319b2d4dc RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress RedisPort: !GetAtt ElastiCache.Outputs.RedisPort MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml index 5b31f9c83..84029987a 100644 --- a/cloudformation/infrastructure/fargate-featurebranch.yaml +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -7,10 +7,6 @@ Parameters: Description: An environment name that will be prefixed to resource names Type: String - PrivateSubnets: - Description: Choose which subnets this ECS cluster should be deployed to - Type: List - SecurityGroup: Description: Select the Security Group to use for the ECS cluster hosts Type: AWS::EC2::SecurityGroup::Id @@ -174,107 +170,6 @@ Resources: ContainerPath: /concordia_images PortMappings: - ContainerPort: 80 - - Name: rabbit - Cpu: 1024 - Memory: 2048 - Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/rabbitmq:${ConcordiaVersion}' - PortMappings: - - ContainerPort: 5672 - LogConfiguration: - LogDriver: awslogs - Options: - awslogs-group: !Ref 'ConcordiaAppLogsGroup' - awslogs-region: !Ref 'AWS::Region' - awslogs-stream-prefix: ConcordiaCron - - Name: importer - Cpu: 1024 - Memory: 2048 - Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia/importer:${ConcordiaVersion}' - LogConfiguration: - LogDriver: awslogs - Options: - awslogs-group: !Ref 'ConcordiaAppLogsGroup' - awslogs-region: !Ref 'AWS::Region' - awslogs-stream-prefix: ConcordiaWorker - Environment: - - Name: AWS - Value: '1' - - Name: ENV_NAME - Value: !Ref EnvName - - Name: CONCORDIA_ENVIRONMENT - Value: !Ref FullEnvironmentName - - Name: S3_BUCKET_NAME - Value: !Ref S3BucketName - - Name: EXPORT_S3_BUCKET_NAME - Value: !Ref ExportS3BucketName - - Name: CELERY_BROKER_URL - Value: pyamqp://guest@localhost:5672 - - Name: AWS_DEFAULT_REGION - Value: !Ref AWS::Region - - Name: SENTRY_BACKEND_DSN - Value: http://34db819263f34c28809da045f841f045@sentry-internal.devops.cloud/2 - - Name: SENTRY_FRONTEND_DSN - Value: https://48ec47f15e484502a29879e40ed2e0c3@crowd-sentry.loc.gov/3 - - Name: REDIS_ADDRESS - Value: !Ref RedisAddress - - Name: REDIS_PORT - Value: !Ref RedisPort - - Name: MEMCACHED_ADDRESS - Value: !Ref MemcachedAddress - - Name: MEMCACHED_PORT - Value: !Ref MemcachedPort - - Name: POSTGRESQL_HOST - Value: !Ref DatabaseEndpoint - - Name: HOST_NAME - Value: !Ref CanonicalHostName - - Name: DJANGO_SETTINGS_MODULE - Value: concordia.settings_ecs - MountPoints: - - SourceVolume: images_volume - ContainerPath: /concordia_images - - Name: celerybeat - Cpu: 1024 - Memory: 2048 - Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia/celerybeat:${ConcordiaVersion}' - LogConfiguration: - LogDriver: awslogs - Options: - awslogs-group: !Ref 'ConcordiaAppLogsGroup' - awslogs-region: !Ref 'AWS::Region' - awslogs-stream-prefix: ConcordiaWorker - Environment: - - Name: AWS - Value: '1' - - Name: ENV_NAME - Value: !Ref EnvName - - Name: CONCORDIA_ENVIRONMENT - Value: !Ref FullEnvironmentName - - Name: S3_BUCKET_NAME - Value: !Ref S3BucketName - - Name: EXPORT_S3_BUCKET_NAME - Value: !Ref ExportS3BucketName - - Name: CELERY_BROKER_URL - Value: pyamqp://guest@localhost:5672 - - Name: AWS_DEFAULT_REGION - Value: !Ref AWS::Region - - Name: SENTRY_BACKEND_DSN - Value: http://34db819263f34c28809da045f841f045@sentry-internal.devops.cloud/2 - - Name: SENTRY_FRONTEND_DSN - Value: https://48ec47f15e484502a29879e40ed2e0c3@crowd-sentry.loc.gov/3 - - Name: REDIS_ADDRESS - Value: !Ref RedisAddress - - Name: REDIS_PORT - Value: !Ref RedisPort - - Name: MEMCACHED_ADDRESS - Value: !Ref MemcachedAddress - - Name: MEMCACHED_PORT - Value: !Ref MemcachedPort - - Name: POSTGRESQL_HOST - Value: !Ref DatabaseEndpoint - - Name: HOST_NAME - Value: !Ref CanonicalHostName - - Name: DJANGO_SETTINGS_MODULE - Value: concordia.settings_ecs ConcordiaExternalService: Type: AWS::ECS::Service @@ -289,7 +184,9 @@ Resources: AwsvpcConfiguration: SecurityGroups: - !Ref SecurityGroup - Subnets: !Ref PrivateSubnets + Subnets: + - subnet-0aa55b322229b945a + - subnet-0f65558b319b2d4dc TaskDefinition: !Ref ConcordiaTask LoadBalancers: - ContainerName: 'app' From 302998f6cdb357acaa54d82b471340f2dfb4fa82 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 12:06:44 -0500 Subject: [PATCH 07/18] Modify cpu and memory values for single container --- cloudformation/infrastructure/fargate-featurebranch.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml index 84029987a..b4a394e52 100644 --- a/cloudformation/infrastructure/fargate-featurebranch.yaml +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -112,8 +112,8 @@ Resources: Type: AWS::ECS::TaskDefinition Properties: Family: !Sub crowd-${EnvName} - Cpu: '4096' - Memory: '16384' + Cpu: '2048' + Memory: '8192' NetworkMode: awsvpc RequiresCompatibilities: - FARGATE From 249a681b806a03dcaf209e6d3ca6c9a8823772c0 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 12:37:07 -0500 Subject: [PATCH 08/18] Move private subnets for elasticache into template file due to CF quirks --- cloudformation/featurebranch.yaml | 29 ++++---- .../infrastructure/elasticache-feature.yaml | 67 +++++++++++++++++++ 2 files changed, 80 insertions(+), 16 deletions(-) create mode 100644 cloudformation/infrastructure/elasticache-feature.yaml diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml index 95ea00f61..fa1c720e1 100644 --- a/cloudformation/featurebranch.yaml +++ b/cloudformation/featurebranch.yaml @@ -20,21 +20,18 @@ Resources: Properties: TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml" Parameters: - DbPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword-rxyUBT:SecretString:password}}' - DatabaseSecurityGroup: sg-0496910b800de2869 - PrivateSubnet1: subnet-0aa55b322229b945a - PrivateSubnet2: subnet-0f65558b319b2d4dc + DbPassword: '{{resolve:secretsmanager:us-east-1:619333082511:secret:crowd/test/DB/MasterUserPassword-rxyUBT:SecretString:password}}' + DatabaseSecurityGroup: 'sg-0496910b800de2869' + PrivateSubnet1: 'subnet-0aa55b322229b945a' + PrivateSubnet2: 'subnet-0f65558b319b2d4dc' ElastiCache: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache.yaml" + TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache-feature.yaml" Parameters: EnvironmentName: !Ref AWS::StackName - SecurityGroup: sg-028ebfe14211447c4 - PrivateSubnets: - - subnet-0aa55b322229b945a - - subnet-0f65558b319b2d4dc + SecurityGroup: 'sg-028ebfe14211447c4' FargateCluster: Type: AWS::CloudFormation::Stack @@ -42,14 +39,14 @@ Resources: TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml" Parameters: EnvironmentName: !Ref AWS::StackName - EnvName: test - FullEnvironmentName: test - S3BucketName: crowd-test-content - ExportS3BucketName: crowd-test-export + EnvName: 'test' + FullEnvironmentName: 'test' + S3BucketName: 'crowd-test-content' + ExportS3BucketName: 'crowd-test-export' ConcordiaVersion: !Ref ConcordiaBranch - CanonicalHostName: !Sub ${ConcordiaBranch}.crowd-test.loc.gov - VpcId: vpc-018e5a73079d0b350 - SecurityGroup: sg-04de21574623caca7 + CanonicalHostName: !Sub '${ConcordiaBranch}.crowd-test.loc.gov' + VpcId: 'vpc-018e5a73079d0b350' + SecurityGroup: 'sg-04de21574623caca7' RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress RedisPort: !GetAtt ElastiCache.Outputs.RedisPort MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress diff --git a/cloudformation/infrastructure/elasticache-feature.yaml b/cloudformation/infrastructure/elasticache-feature.yaml new file mode 100644 index 000000000..eb7f2a86b --- /dev/null +++ b/cloudformation/infrastructure/elasticache-feature.yaml @@ -0,0 +1,67 @@ +Description: > + This template deploys an elasticache cluster to the provided VPC and subnets + +Parameters: + + EnvironmentName: + Description: An environment name that will be prefixed to resource names + Type: String + + SecurityGroup: + Description: Select the Security Group to use for the ECS cluster hosts + Type: AWS::EC2::SecurityGroup::Id + + CacheNodeType: + Type: String + Default: cache.m1.small + +Resources: + + CachePrivateSubnetGroup: + Type: AWS::ElastiCache::SubnetGroup + Properties: + Description: Private subnet group + SubnetIds: + - 'subnet-0aa55b322229b945a' + - 'subnet-0f65558b319b2d4dc' + MemcachedService: + Type: AWS::ElastiCache::CacheCluster + Properties: + VpcSecurityGroupIds: + - !Ref 'SecurityGroup' + CacheSubnetGroupName: !Ref 'CachePrivateSubnetGroup' + CacheNodeType: !Ref 'CacheNodeType' + ClusterName: !Sub '${EnvironmentName}-cache' + Engine: memcached + AutoMinorVersionUpgrade: true + NumCacheNodes: 1 + RedisService: + Type: AWS::ElastiCache::CacheCluster + Properties: + VpcSecurityGroupIds: + - !Ref 'SecurityGroup' + CacheSubnetGroupName: !Ref 'CachePrivateSubnetGroup' + CacheNodeType: !Ref 'CacheNodeType' + ClusterName: !Sub '${EnvironmentName}-redis' + Engine: redis + AutoMinorVersionUpgrade: true + NumCacheNodes: 1 + SnapshotRetentionLimit: 1 + +Outputs: + + RedisAddress: + Description: Redis endpoint address + Value: !GetAtt 'RedisService.RedisEndpoint.Address' + + RedisPort: + Description: Redis endpoint port + Value: !GetAtt 'RedisService.RedisEndpoint.Port' + + MemcachedAddress: + Description: memcached endpoint address + Value: !GetAtt 'MemcachedService.ConfigurationEndpoint.Address' + + MemcachedPort: + Description: memcached endpoint port + Value: !GetAtt 'MemcachedService.ConfigurationEndpoint.Port' From ba1bf9a318cc51124d30d9c5a8cf142b2054f62a Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 13:07:05 -0500 Subject: [PATCH 09/18] Use abbreviated name to avoid exceeding char limits on elasticache services, task definition named with feature branch --- cloudformation/featurebranch.yaml | 11 ++++++++--- .../infrastructure/fargate-featurebranch.yaml | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml index fa1c720e1..9f99232e2 100644 --- a/cloudformation/featurebranch.yaml +++ b/cloudformation/featurebranch.yaml @@ -12,6 +12,11 @@ Parameters: Description: which branch name to deploy Type: String Default: release + + AbbreviatedName: + Description: an abbreviation used for creating cloudformation Resources + Type: String + Default: rel Resources: @@ -20,7 +25,7 @@ Resources: Properties: TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml" Parameters: - DbPassword: '{{resolve:secretsmanager:us-east-1:619333082511:secret:crowd/test/DB/MasterUserPassword-rxyUBT:SecretString:password}}' + DbPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword:SecretString:password}}' DatabaseSecurityGroup: 'sg-0496910b800de2869' PrivateSubnet1: 'subnet-0aa55b322229b945a' PrivateSubnet2: 'subnet-0f65558b319b2d4dc' @@ -30,7 +35,7 @@ Resources: Properties: TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache-feature.yaml" Parameters: - EnvironmentName: !Ref AWS::StackName + EnvironmentName: !Ref AbbreviatedName SecurityGroup: 'sg-028ebfe14211447c4' FargateCluster: @@ -38,7 +43,7 @@ Resources: Properties: TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml" Parameters: - EnvironmentName: !Ref AWS::StackName + EnvironmentName: !Ref AbbreviatedName EnvName: 'test' FullEnvironmentName: 'test' S3BucketName: 'crowd-test-content' diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml index b4a394e52..49934fbdd 100644 --- a/cloudformation/infrastructure/fargate-featurebranch.yaml +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -111,7 +111,7 @@ Resources: ConcordiaTask: Type: AWS::ECS::TaskDefinition Properties: - Family: !Sub crowd-${EnvName} + Family: !Sub crowd-${ConcordiaVersion} Cpu: '2048' Memory: '8192' NetworkMode: awsvpc From 5f7fd4533a1d70689f90260e285a32f81c1e6bd4 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 13:18:10 -0500 Subject: [PATCH 10/18] Use crowd-test task role for feature branch deployments --- cloudformation/featurebranch.yaml | 1 - cloudformation/infrastructure/fargate-featurebranch.yaml | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml index 9f99232e2..dd2c9b60b 100644 --- a/cloudformation/featurebranch.yaml +++ b/cloudformation/featurebranch.yaml @@ -43,7 +43,6 @@ Resources: Properties: TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml" Parameters: - EnvironmentName: !Ref AbbreviatedName EnvName: 'test' FullEnvironmentName: 'test' S3BucketName: 'crowd-test-content' diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml index 49934fbdd..b037da5fb 100644 --- a/cloudformation/infrastructure/fargate-featurebranch.yaml +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -3,10 +3,6 @@ Description: > Parameters: - EnvironmentName: - Description: An environment name that will be prefixed to resource names - Type: String - SecurityGroup: Description: Select the Security Group to use for the ECS cluster hosts Type: AWS::EC2::SecurityGroup::Id @@ -118,7 +114,7 @@ Resources: RequiresCompatibilities: - FARGATE ExecutionRoleArn: ecsTaskExecutionRole - TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/ConcordiaServerTaskRole-${EnvironmentName}' + TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/ConcordiaServerTaskRole-crowd-test' Volumes: - Name: images_volume ContainerDefinitions: From da824be237101ee24c8b4b7d32068c9afc5325a9 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 13:44:45 -0500 Subject: [PATCH 11/18] Correct import path for get_random_secret_key --- concordia/settings_docker.py | 2 +- concordia/settings_ecs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/concordia/settings_docker.py b/concordia/settings_docker.py index e4e779d5f..3aadcc097 100644 --- a/concordia/settings_docker.py +++ b/concordia/settings_docker.py @@ -1,6 +1,6 @@ import os -from django.core.management import get_random_secret_key +from django.core.management.utils import get_random_secret_key from .settings_template import * # NOQA ignore=F405 from .settings_template import INSTALLED_APPS, LOGGING diff --git a/concordia/settings_ecs.py b/concordia/settings_ecs.py index 1b249abc8..c09bf5f4d 100644 --- a/concordia/settings_ecs.py +++ b/concordia/settings_ecs.py @@ -1,7 +1,7 @@ import json import os -from django.core.management import get_random_secret_key +from django.core.management.utils import get_random_secret_key from .secrets import get_secret from .settings_template import * # NOQA ignore=F405 From 8960929472bc15eecfcb4dc23f4adefdc6828b87 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 14:13:18 -0500 Subject: [PATCH 12/18] Fix bug in initial site config management command --- .../management/commands/ensure_initial_site_configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/concordia/management/commands/ensure_initial_site_configuration.py b/concordia/management/commands/ensure_initial_site_configuration.py index f4b6e5468..7f8af297f 100644 --- a/concordia/management/commands/ensure_initial_site_configuration.py +++ b/concordia/management/commands/ensure_initial_site_configuration.py @@ -72,7 +72,7 @@ def handle(self, *, admin_username, admin_email, site_name, site_domain, **optio ) if site_domain != "example.com": - updated = Site.objects.filter.update(name=site_name, domain=site_domain) + updated = Site.objects.update(name=site_name, domain=site_domain) if updated: self.stdout.write( f"Configured site with name {site_name} and domain {site_domain}" From 2f6a2fd4a5f208e9ddb1b6b0c7ce3a62fcd61a84 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 14:35:18 -0500 Subject: [PATCH 13/18] Remove unneeded parameter --- cloudformation/infrastructure/fargate-cluster.yaml | 6 +----- cloudformation/master.yaml | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cloudformation/infrastructure/fargate-cluster.yaml b/cloudformation/infrastructure/fargate-cluster.yaml index 49b2748a2..e7cab64f9 100644 --- a/cloudformation/infrastructure/fargate-cluster.yaml +++ b/cloudformation/infrastructure/fargate-cluster.yaml @@ -71,10 +71,6 @@ Parameters: Type: String Description: canonical host name of the application, e.g. crowd-test.loc.gov - CertificateName: - Type: String - Description: Normally the canonical host name, except for subdomains in crowd-test.loc.gov - DatabaseEndpoint: Type: String Description: Host name of the Postgres RDS service @@ -206,7 +202,7 @@ Resources: SecureExternalLoadBalancerListener: Properties: Certificates: - - CertificateArn: !Sub 'arn:aws:iam::${AWS::AccountId}:server-certificate/${CertificateName}' + - CertificateArn: !Sub 'arn:aws:iam::${AWS::AccountId}:server-certificate/${CanonicalHostName}' DefaultActions: - TargetGroupArn: !Ref ConcordiaExternalTargetGroup Type: forward diff --git a/cloudformation/master.yaml b/cloudformation/master.yaml index f4a73058e..76c5563da 100644 --- a/cloudformation/master.yaml +++ b/cloudformation/master.yaml @@ -175,7 +175,6 @@ Resources: MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort CanonicalHostName: !Ref CanonicalHostName - CertificateName: !Ref CanonicalHostName DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName FullEnvironmentName: !Ref FullEnvironmentName S3BucketName: !FindInMap [EnvironmentMapping, S3BucketNameMap, !Ref EnvName] From cb12e85c8a23dc51e2ff00fd6a81ede57757af61 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Mon, 14 Jan 2019 15:36:17 -0500 Subject: [PATCH 14/18] Remove unused deployment code --- cloudformation/update_services.sh | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100755 cloudformation/update_services.sh diff --git a/cloudformation/update_services.sh b/cloudformation/update_services.sh deleted file mode 100755 index 4a63dd3d2..000000000 --- a/cloudformation/update_services.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -eu - -# DEV ENVIRONMENT -# CLUSTER_NAME=crowd-dev -# SERVICE_NAME=crowd-dev-app-Service-PPFPP27E26LE - -# TEST ENVIRONMENT -# CLUSTER_NAME=crowd-test2 -# SERVICE_NAME=crowd-test-app-Service-KRK54VT16MJN - -# STAGE ENVIRONMENT -CLUSTER_NAME=crowd-stage -SERVICE_NAME=crowd-stage-app-Service-EV9ZR04V07M1 - -# PROD ENVIRONMENT -# CLUSTER_NAME=crowd-prod -# SERVICE_NAME=crowd-prod-app-Service - -AWS_REGION=us-east-1 - -export CLUSTER_NAME AWS_REGION SERVICE_NAME - -aws ecs update-service --region $AWS_REGION --cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment From 7ac1ebbff9a53ba62ae49ea690a743698e65d7a2 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Tue, 15 Jan 2019 09:30:26 -0500 Subject: [PATCH 15/18] As long as feature branch has its own elasticache template, simplify hardcoded values --- .../infrastructure/elasticache-feature.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/cloudformation/infrastructure/elasticache-feature.yaml b/cloudformation/infrastructure/elasticache-feature.yaml index eb7f2a86b..3c278cb70 100644 --- a/cloudformation/infrastructure/elasticache-feature.yaml +++ b/cloudformation/infrastructure/elasticache-feature.yaml @@ -17,19 +17,12 @@ Parameters: Resources: - CachePrivateSubnetGroup: - Type: AWS::ElastiCache::SubnetGroup - Properties: - Description: Private subnet group - SubnetIds: - - 'subnet-0aa55b322229b945a' - - 'subnet-0f65558b319b2d4dc' MemcachedService: Type: AWS::ElastiCache::CacheCluster Properties: VpcSecurityGroupIds: - !Ref 'SecurityGroup' - CacheSubnetGroupName: !Ref 'CachePrivateSubnetGroup' + CacheSubnetGroupName: 'crowd-cache-1frtjeewr57u7' CacheNodeType: !Ref 'CacheNodeType' ClusterName: !Sub '${EnvironmentName}-cache' Engine: memcached @@ -40,7 +33,7 @@ Resources: Properties: VpcSecurityGroupIds: - !Ref 'SecurityGroup' - CacheSubnetGroupName: !Ref 'CachePrivateSubnetGroup' + CacheSubnetGroupName: 'crowd-cache-1frtjeewr57u7' CacheNodeType: !Ref 'CacheNodeType' ClusterName: !Sub '${EnvironmentName}-redis' Engine: redis From 6d302768194c41ae2fe8cdf728ca6bdded79ec25 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Tue, 15 Jan 2019 09:33:17 -0500 Subject: [PATCH 16/18] Clarify description of concordiaversion for feature branch deploys --- cloudformation/infrastructure/fargate-featurebranch.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml index b037da5fb..e95aaf342 100644 --- a/cloudformation/infrastructure/fargate-featurebranch.yaml +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -13,7 +13,7 @@ Parameters: ConcordiaVersion: Type: String - Description: version of concordia, concordia/importer, and rabbitmq docker images to pull and deploy + Description: docker tag of concordia app image to pull and deploy Default: latest EnvName: From 138d0ff415490fff4e73fd3fec0791f90cde89d0 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Tue, 15 Jan 2019 09:37:05 -0500 Subject: [PATCH 17/18] Unify quotes in featurebranch.yaml --- cloudformation/featurebranch.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml index dd2c9b60b..12e17bd93 100644 --- a/cloudformation/featurebranch.yaml +++ b/cloudformation/featurebranch.yaml @@ -2,7 +2,7 @@ AWSTemplateFormatVersion: '2010-09-09' Description: > Deploy a feature branch to a subdomain of crowd-test.loc.gov - using pre-existing infrastructure + using pre-existing infrastructure. Assumes docker images have been published to ECR with tag matching the feature branch name. @@ -14,7 +14,7 @@ Parameters: Default: release AbbreviatedName: - Description: an abbreviation used for creating cloudformation Resources + Description: an abbreviation used for creating short-named cloudformation resources Type: String Default: rel @@ -23,7 +23,7 @@ Resources: RDS: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml" + TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml' Parameters: DbPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword:SecretString:password}}' DatabaseSecurityGroup: 'sg-0496910b800de2869' @@ -33,7 +33,7 @@ Resources: ElastiCache: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache-feature.yaml" + TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache-feature.yaml' Parameters: EnvironmentName: !Ref AbbreviatedName SecurityGroup: 'sg-028ebfe14211447c4' @@ -41,7 +41,7 @@ Resources: FargateCluster: Type: AWS::CloudFormation::Stack Properties: - TemplateURL: "https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml" + TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml' Parameters: EnvName: 'test' FullEnvironmentName: 'test' From d07deb62fa4fd96fa1ebd2f285a9776623618656 Mon Sep 17 00:00:00 2001 From: Rosie Storey Date: Thu, 17 Jan 2019 11:22:55 -0500 Subject: [PATCH 18/18] Clean up container build script that had messy merge --- build_containers.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/build_containers.sh b/build_containers.sh index d39f4c762..dca9a4d0d 100755 --- a/build_containers.sh +++ b/build_containers.sh @@ -6,10 +6,6 @@ BUILD_ALL=${BUILD_ALL:=0} BUILD_NUMBER=${BUILD_NUMBER:=1} TAG=${TAG:-test} -BUILD_ALL=${BUILD_ALL:=0} -BUILD_NUMBER=${BUILD_NUMBER:=1} -TAG=${TAG:-test} - # Get an unique venv folder to using *inside* workspace VENV=".venv-${BUILD_NUMBER}" @@ -33,13 +29,11 @@ eval "$(aws ecr get-login --no-include-email --region us-east-1)" python3 setup.py build docker build -t concordia . - docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}" docker tag concordia:latest "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}" docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${VERSION_NUMBER}" docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia:${TAG}" - if [ $BUILD_ALL -eq 1 ]; then docker build -t concordia/importer --file importer/Dockerfile . @@ -67,4 +61,3 @@ if [ $BUILD_ALL -eq 1 ]; then docker push "${AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com/concordia/indexer:${TAG}" fi -