diff --git a/cloudformation/featurebranch.yaml b/cloudformation/featurebranch.yaml new file mode 100644 index 000000000..12e17bd93 --- /dev/null +++ b/cloudformation/featurebranch.yaml @@ -0,0 +1,58 @@ +--- +AWSTemplateFormatVersion: '2010-09-09' +Description: > + Deploy a feature branch to a subdomain of crowd-test.loc.gov + using pre-existing infrastructure. + Assumes docker images have been published to ECR with + tag matching the feature branch name. + +Parameters: + + ConcordiaBranch: + Description: which branch name to deploy + Type: String + Default: release + + AbbreviatedName: + Description: an abbreviation used for creating short-named cloudformation resources + Type: String + Default: rel + +Resources: + + RDS: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/rds.yaml' + Parameters: + DbPassword: '{{resolve:secretsmanager:crowd/test/DB/MasterUserPassword:SecretString:password}}' + DatabaseSecurityGroup: 'sg-0496910b800de2869' + PrivateSubnet1: 'subnet-0aa55b322229b945a' + PrivateSubnet2: 'subnet-0f65558b319b2d4dc' + + ElastiCache: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/elasticache-feature.yaml' + Parameters: + EnvironmentName: !Ref AbbreviatedName + SecurityGroup: 'sg-028ebfe14211447c4' + + FargateCluster: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: 'https://s3.amazonaws.com/crowd-deployment/infrastructure/fargate-featurebranch.yaml' + Parameters: + EnvName: 'test' + FullEnvironmentName: 'test' + S3BucketName: 'crowd-test-content' + ExportS3BucketName: 'crowd-test-export' + ConcordiaVersion: !Ref ConcordiaBranch + CanonicalHostName: !Sub '${ConcordiaBranch}.crowd-test.loc.gov' + VpcId: 'vpc-018e5a73079d0b350' + SecurityGroup: 'sg-04de21574623caca7' + RedisAddress: !GetAtt ElastiCache.Outputs.RedisAddress + RedisPort: !GetAtt ElastiCache.Outputs.RedisPort + MemcachedAddress: !GetAtt ElastiCache.Outputs.MemcachedAddress + MemcachedPort: !GetAtt ElastiCache.Outputs.MemcachedPort + DatabaseEndpoint: !GetAtt RDS.Outputs.DatabaseHostName diff --git a/cloudformation/infrastructure/elasticache-feature.yaml b/cloudformation/infrastructure/elasticache-feature.yaml new file mode 100644 index 000000000..3c278cb70 --- /dev/null +++ b/cloudformation/infrastructure/elasticache-feature.yaml @@ -0,0 +1,60 @@ +Description: > + This template deploys an elasticache cluster to the provided VPC and subnets + +Parameters: + + EnvironmentName: + Description: An environment name that will be prefixed to resource names + Type: String + + SecurityGroup: + Description: Select the Security Group to use for the ECS cluster hosts + Type: AWS::EC2::SecurityGroup::Id + + CacheNodeType: + Type: String + Default: cache.m1.small + +Resources: + + MemcachedService: + Type: AWS::ElastiCache::CacheCluster + Properties: + VpcSecurityGroupIds: + - !Ref 'SecurityGroup' + CacheSubnetGroupName: 'crowd-cache-1frtjeewr57u7' + CacheNodeType: !Ref 'CacheNodeType' + ClusterName: !Sub '${EnvironmentName}-cache' + Engine: memcached + AutoMinorVersionUpgrade: true + NumCacheNodes: 1 + RedisService: + Type: AWS::ElastiCache::CacheCluster + Properties: + VpcSecurityGroupIds: + - !Ref 'SecurityGroup' + CacheSubnetGroupName: 'crowd-cache-1frtjeewr57u7' + CacheNodeType: !Ref 'CacheNodeType' + ClusterName: !Sub '${EnvironmentName}-redis' + Engine: redis + AutoMinorVersionUpgrade: true + NumCacheNodes: 1 + SnapshotRetentionLimit: 1 + +Outputs: + + RedisAddress: + Description: Redis endpoint address + Value: !GetAtt 'RedisService.RedisEndpoint.Address' + + RedisPort: + Description: Redis endpoint port + Value: !GetAtt 'RedisService.RedisEndpoint.Port' + + MemcachedAddress: + Description: memcached endpoint address + Value: !GetAtt 'MemcachedService.ConfigurationEndpoint.Address' + + MemcachedPort: + Description: memcached endpoint port + Value: !GetAtt 'MemcachedService.ConfigurationEndpoint.Port' diff --git a/cloudformation/infrastructure/fargate-cluster.yaml b/cloudformation/infrastructure/fargate-cluster.yaml index e61e06667..e7cab64f9 100644 --- a/cloudformation/infrastructure/fargate-cluster.yaml +++ b/cloudformation/infrastructure/fargate-cluster.yaml @@ -219,7 +219,7 @@ Resources: ConcordiaTask: Type: AWS::ECS::TaskDefinition Properties: - Family: !Sub concordia-${EnvName} + Family: !Sub crowd-${EnvName} Cpu: '4096' Memory: '16384' NetworkMode: awsvpc diff --git a/cloudformation/infrastructure/fargate-featurebranch.yaml b/cloudformation/infrastructure/fargate-featurebranch.yaml new file mode 100644 index 000000000..e95aaf342 --- /dev/null +++ b/cloudformation/infrastructure/fargate-featurebranch.yaml @@ -0,0 +1,190 @@ +Description: > + This template deploys a fargate cluster to the provided VPC and subnets + +Parameters: + + SecurityGroup: + Description: Select the Security Group to use for the ECS cluster hosts + Type: AWS::EC2::SecurityGroup::Id + + VpcId: + Description: The Id of the VPC for this cluster + Type: AWS::EC2::VPC::Id + + ConcordiaVersion: + Type: String + Description: docker tag of concordia app image to pull and deploy + Default: latest + + EnvName: + Type: String + Description: which environment to target + AllowedValues: + - 'dev' + - 'test' + - 'stage' + - 'prod' + ConstraintDescription: Must match a location for secret storage in secretsmanager + + FullEnvironmentName: + Type: String + Description: Full name of deployment environment + AllowedValues: + - 'development' + - 'test' + - 'staging' + - 'production' + + RedisAddress: + Type: String + Description: Redis endpoint address + + RedisPort: + Type: String + Description: Redis endpoint port + + MemcachedAddress: + Type: String + Description: memcached endpoint address + + MemcachedPort: + Type: String + Description: memcached endpoint port + + CanonicalHostName: + Type: String + Description: canonical host name of the application, e.g. crowd-test.loc.gov + + DatabaseEndpoint: + Type: String + Description: Host name of the Postgres RDS service + + S3BucketName: + Type: String + Description: name of the S3 bucket (public) where collection images will be stored + + ExportS3BucketName: + Type: String + Description: name of the S3 bucket (public) where exported transcriptions will be stored + + +Resources: + + ConcordiaAppLogsGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Ref AWS::StackName + RetentionInDays: 30 + + + ConcordiaExternalTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + HealthCheckIntervalSeconds: 30 + HealthCheckPath: /healthz + HealthCheckProtocol: HTTP + HealthCheckTimeoutSeconds: 5 + HealthyThresholdCount: 2 + UnhealthyThresholdCount: 10 + TargetType: ip + Port: 80 + Protocol: HTTP + VpcId: !Ref VpcId + + SubdomainListenerRule: + Type: AWS::ElasticLoadBalancingV2::ListenerRule + Properties: + Actions: + - TargetGroupArn: !Ref ConcordiaExternalTargetGroup + Type: forward + Conditions: + - Field: host-header + Values: + - !Ref CanonicalHostName + ListenerArn: arn:aws:elasticloadbalancing:us-east-1:619333082511:listener/app/crowd-test/81e4820e354ea810/187fd94e534ad833 + Priority: 100 + + ConcordiaTask: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub crowd-${ConcordiaVersion} + Cpu: '2048' + Memory: '8192' + NetworkMode: awsvpc + RequiresCompatibilities: + - FARGATE + ExecutionRoleArn: ecsTaskExecutionRole + TaskRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/ConcordiaServerTaskRole-crowd-test' + Volumes: + - Name: images_volume + ContainerDefinitions: + - Name: app + Cpu: 2048 + Memory: 8192 + Image: !Sub '${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/concordia:${ConcordiaVersion}' + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref 'ConcordiaAppLogsGroup' + awslogs-region: !Ref 'AWS::Region' + awslogs-stream-prefix: ConcordiaServer + Environment: + - Name: AWS + Value: '1' + - Name: ENV_NAME + Value: !Ref EnvName + - Name: CONCORDIA_ENVIRONMENT + Value: !Ref FullEnvironmentName + - Name: S3_BUCKET_NAME + Value: !Ref S3BucketName + - Name: EXPORT_S3_BUCKET_NAME + Value: !Ref ExportS3BucketName + - Name: CELERY_BROKER_URL + Value: pyamqp://guest@localhost:5672 + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: SENTRY_BACKEND_DSN + Value: http://34db819263f34c28809da045f841f045@sentry-internal.devops.cloud/2 + - Name: SENTRY_FRONTEND_DSN + Value: https://48ec47f15e484502a29879e40ed2e0c3@crowd-sentry.loc.gov/3 + - Name: REDIS_ADDRESS + Value: !Ref RedisAddress + - Name: REDIS_PORT + Value: !Ref RedisPort + - Name: MEMCACHED_ADDRESS + Value: !Ref MemcachedAddress + - Name: MEMCACHED_PORT + Value: !Ref MemcachedPort + - Name: POSTGRESQL_HOST + Value: !Ref DatabaseEndpoint + - Name: HOST_NAME + Value: !Ref CanonicalHostName + - Name: DJANGO_SETTINGS_MODULE + Value: concordia.settings_ecs + MountPoints: + - SourceVolume: images_volume + ContainerPath: /concordia_images + PortMappings: + - ContainerPort: 80 + + ConcordiaExternalService: + Type: AWS::ECS::Service + Properties: + Cluster: crowd-test + LaunchType: FARGATE + DeploymentConfiguration: + MaximumPercent: 200 + MinimumHealthyPercent: 75 + DesiredCount: 1 + NetworkConfiguration: + AwsvpcConfiguration: + SecurityGroups: + - !Ref SecurityGroup + Subnets: + - subnet-0aa55b322229b945a + - subnet-0f65558b319b2d4dc + TaskDefinition: !Ref ConcordiaTask + LoadBalancers: + - ContainerName: 'app' + ContainerPort: 80 + TargetGroupArn: !Ref ConcordiaExternalTargetGroup \ No newline at end of file diff --git a/cloudformation/master.yaml b/cloudformation/master.yaml index 8ff92b192..76c5563da 100644 --- a/cloudformation/master.yaml +++ b/cloudformation/master.yaml @@ -7,10 +7,10 @@ Description: > route on the public subnets. It deploys a pair of NAT Gateways (one in each AZ), and default routes for them in the private subnets. - It then deploys a highly available ECS cluster using an AutoScaling Group, with - ECS hosts distributed across multiple Availability Zones. + It then deploys a Fargate ECS cluster distributed across multiple + Availability Zones. - Finally, it deploys a pair of example ECS services from containers published in + Finally, it deploys crowd ECS services from containers published in Amazon EC2 Container Registry (Amazon ECR). Mappings: EnvironmentMapping: diff --git a/cloudformation/setup.sh b/cloudformation/setup.sh deleted file mode 100644 index 2c25ab12d..000000000 --- a/cloudformation/setup.sh +++ /dev/null @@ -1,14 +0,0 @@ -# export AWS_ACCESS_KEY="" -# export AWS_SECRET_KEY="" -# export AWS_CREDENTIAL_FILE="" - -# Make sure your aws credentials are configured before attempting to run this script. - -S3_BUCKET_NAME = "rstorey-concordia-refarch" -STACK_NAME = "rstorey-refarch-test" - -aws s3api create-bucket --bucket $S3_BUCKET_NAME --region us-east-1 - -aws s3 sync . s3://$S3_BUCKET_NAME - -aws cloudformation create-stack --stack-name $STACK_NAME --template-url https://s3.amazonaws.com/$S3_BUCKET_NAME/master.yaml \ No newline at end of file diff --git a/cloudformation/update_services.sh b/cloudformation/update_services.sh deleted file mode 100755 index 4a63dd3d2..000000000 --- a/cloudformation/update_services.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -eu - -# DEV ENVIRONMENT -# CLUSTER_NAME=crowd-dev -# SERVICE_NAME=crowd-dev-app-Service-PPFPP27E26LE - -# TEST ENVIRONMENT -# CLUSTER_NAME=crowd-test2 -# SERVICE_NAME=crowd-test-app-Service-KRK54VT16MJN - -# STAGE ENVIRONMENT -CLUSTER_NAME=crowd-stage -SERVICE_NAME=crowd-stage-app-Service-EV9ZR04V07M1 - -# PROD ENVIRONMENT -# CLUSTER_NAME=crowd-prod -# SERVICE_NAME=crowd-prod-app-Service - -AWS_REGION=us-east-1 - -export CLUSTER_NAME AWS_REGION SERVICE_NAME - -aws ecs update-service --region $AWS_REGION --cluster $CLUSTER_NAME --service $SERVICE_NAME --force-new-deployment diff --git a/concordia/settings_ecs.py b/concordia/settings_ecs.py index 5230191fb..c09bf5f4d 100644 --- a/concordia/settings_ecs.py +++ b/concordia/settings_ecs.py @@ -1,6 +1,8 @@ import json import os +from django.core.management.utils import get_random_secret_key + from .secrets import get_secret from .settings_template import * # NOQA ignore=F405 from .settings_template import CONCORDIA_ENVIRONMENT, DATABASES, INSTALLED_APPS, LOGGING @@ -32,7 +34,7 @@ EMAIL_HOST_PASSWORD = smtp_secret["Password"] else: - DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", "changeme") + DJANGO_SECRET_KEY = os.getenv("DJANGO_SECRET_KEY", get_random_secret_key()) EMAIL_HOST = os.environ.get("EMAIL_HOST", "localhost") EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER", "") EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD", "")