From 769643a6e290444721968ecd1b123bf0293539f9 Mon Sep 17 00:00:00 2001 From: Matt Webster Date: Tue, 1 Dec 2020 10:09:59 -0800 Subject: [PATCH] added vacuum setting and task fixes --- .github/workflows/deploy_backend_dev.yml | 18 ------------------ server/aws/README.md | 21 +++++++++++++++------ server/aws/templates/prefect.json | 4 +--- server/aws/variables.tf | 4 ++-- server/prefect/config.toml | 8 ++------ server/prefect/tasks/postgres.py | 9 +++++---- 6 files changed, 25 insertions(+), 39 deletions(-) delete mode 100644 .github/workflows/deploy_backend_dev.yml diff --git a/.github/workflows/deploy_backend_dev.yml b/.github/workflows/deploy_backend_dev.yml deleted file mode 100644 index f221dee44..000000000 --- a/.github/workflows/deploy_backend_dev.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: deploy_backend_dev -on: - push: - branches: - - dev - paths: - - 'server/**' -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Deploy Amazon ECS task definition - uses: aws-actions/amazon-ecs-deploy-task-definition@v1 - with: - task-definition: server/task-definition.json - service: api-service - cluster: dev-cluster - wait-for-service-stability: true diff --git a/server/aws/README.md b/server/aws/README.md index 07e903de8..813cd5138 100644 --- a/server/aws/README.md +++ b/server/aws/README.md @@ -14,22 +14,25 @@ Note that this blueprint only handles the server infrastructure. The client appl ## Assumptions -- AWS account is already created -- A task-definition.json is already loaded to the AWS account +- AWS account is already created and profile with required user credentials set up locally - IAM Role (ecsTaskExecutionRole) already created with SSMReadOnlyAccess AND ECSTaskExecutionRolePolicy applied - SSL/TLS certificate is already created and loaded to the AWS account -- DNS is manually pointed to the ALB once the deployment is complete +- DNS can be manually pointed to the ALB once the deployment is complete ## Deployment The blueprint is meant to be deployed manually. The person deploying the blueprint will have the AWS CLI installed and a profile with Admin access for the AWS account being used. Use the Terraform commands (init, plan, apply, etc.) to configure the 0.12 version environment and deploy the environment. +The same blueprint is intended to be run in separate stages for ```dev``` and ```prod``` using Terraform Workspaces as described [here](https://learn.hashicorp.com/tutorials/terraform/organize-configuration?in=terraform/modules#separate-states). + ### Parameters -The following parameters (at minimum) need to be set in order to run the blueprint. You can use, for example, a .tfvars file for these. +The following parameters (at minimum) need to be set in order to run the blueprint. You can use, for example, a .tfvars file for these (e.g. a dev.tfvars and prod.tfvars) - profile - account_id +- stage +- image_tag - db_name - db_username - db_password @@ -43,9 +46,15 @@ For example, the value from ```/dev/us-east-1/DB_DSN``` will be injected as the ## After the Deployment -The environment will be created with a blank database. As a result the application will initially show an error message. The initial Alembic migration needs to be run then the database populated using the Prefect data pipeline. +The environment will be created with a blank database. As a result the application will initially show an error message. The initial Alembic migration needs to be run then the database populated using the Prefect data pipeline. The easiest way to do this is with a SSH tunnel to the database using the Bastion server. + +Alternatively, the appropriate ECS task ```prod-la-311-data-server``` can be run manually with the following container Command override: + +```bash +alembic,upgrade,head +``` -The easiest way to do this is with a SSH tunnel to the database using the Bastion server. +Once the database schema has been applied the data loading task ```prod-la-311-data-nightly-update``` can be manually run. When run on a blank database it will pull all available data. ### Developer Access with SSH and Bastion Server diff --git a/server/aws/templates/prefect.json b/server/aws/templates/prefect.json index f0a1aea6b..5398f6653 100644 --- a/server/aws/templates/prefect.json +++ b/server/aws/templates/prefect.json @@ -13,9 +13,7 @@ }, "secrets": [ { "name": "PREFECT__CONTEXT__SECRETS__DSN", "valueFrom": "/${stage}/${region}/DB_DSN" }, - { "name": "PREFECT__CONTEXT__SECRETS__SLACK_HOOK", "valueFrom": "/${stage}/${region}/SLACK_HOOK" } - ], - "environment": [ + { "name": "PREFECT__CONTEXT__SECRETS__SLACK_HOOK", "valueFrom": "/${stage}/${region}/SLACK_HOOK" }, { "name": "PREFECT__API_URL", "valueFrom": "/${stage}/${region}/API_URL" }, { "name": "PREFECT__STAGE", "valueFrom": "/${stage}/${region}/STAGE" } ], diff --git a/server/aws/variables.tf b/server/aws/variables.tf index e62ffef53..367553e7c 100644 --- a/server/aws/variables.tf +++ b/server/aws/variables.tf @@ -36,12 +36,12 @@ variable db_password {} variable container_cpu { type = number - default = 256 + default = 512 } variable container_memory { type = number - default = 1024 + default = 2048 } variable container_port { diff --git a/server/prefect/config.toml b/server/prefect/config.toml index 87ae4363c..1a7980e70 100644 --- a/server/prefect/config.toml +++ b/server/prefect/config.toml @@ -1,22 +1,18 @@ # Default configuration for 311 Data prefect task - # These settings can be overridden with environment variables that follow the pattern: # PREFECT__[setting name] - # determines whether Dask should be used to parallelize the flow run dask = true - # (!) WARNING: setting this to "true" will wipe out whatever is in the target table (e.g. 'requests') reset_db = false - # will write to ./output if left blank but can be overridden temp_folder = "" - # set to local API instance or override with PREFECT__API_URL api_url = "http://localhost:5000" - # whether the data update is being run in a testing, development or production environment stage = "Testing" +# whether to vacuum database and reset stats after load +vacuum_db = false [socrata] host = "data.lacity.org" diff --git a/server/prefect/tasks/postgres.py b/server/prefect/tasks/postgres.py index 128189b17..6898eb5d7 100644 --- a/server/prefect/tasks/postgres.py +++ b/server/prefect/tasks/postgres.py @@ -214,10 +214,11 @@ def complete_load() -> Dict[str, int]: connection.commit() logger.info("Views successfully refreshed") - # need to have autocommit set for VACUUM to work - connection.autocommit = True - cursor.execute("VACUUM FULL ANALYZE") - logger.info("Database vacuumed and analyzed") + if prefect.config.vacuum_db: + # need to have autocommit set for VACUUM to work + connection.autocommit = True + cursor.execute("VACUUM FULL ANALYZE") + logger.info("Database vacuumed and analyzed") cursor.close() connection.close()