From e71ef1a3ce61a1d759b6d3057f03c189f6d21ae7 Mon Sep 17 00:00:00 2001 From: Derek Dohler Date: Wed, 14 Jun 2023 13:38:54 -0400 Subject: [PATCH 1/3] Upgrade Troposphere to version 3 This broke majorkirby because it uses `add_version` in its `set_up_stack()` method, which was changed to set_version() in Troposphere v3. To work around the issue without pushing a new release to majorkirby, this removes calls to super() so that the code using add_version() doesn't get called and then calls set_version() in local project code so that functionality is preserved. --- deployment/cloudformation/app.py | 4 ++-- deployment/cloudformation/data.py | 4 ++-- deployment/cloudformation/vpc.py | 4 ++-- python/cac_tripplanner/deployment_requirements.txt | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/deployment/cloudformation/app.py b/deployment/cloudformation/app.py index a04be0de0..b2a99abe5 100644 --- a/deployment/cloudformation/app.py +++ b/deployment/cloudformation/app.py @@ -54,10 +54,10 @@ def set_up_stack(self): if not self.INPUTS or not self.STACK_NAME_PREFIX or not self.HEALTH_ENDPOINT: raise MKInputError('Must define INPUTS, STACK_NAME_PREFIX, and HEALTH_ENDPOINT') - super(AppServerStack, self).set_up_stack() + self.set_version() tags = self.get_input('Tags').copy() - self.add_description('{} App Server Stack for Cac'.format(self.STACK_NAME_PREFIX)) + self.set_description('{} App Server Stack for Cac'.format(self.STACK_NAME_PREFIX)) assert isinstance(tags, dict), 'tags must be a dictionary' diff --git a/deployment/cloudformation/data.py b/deployment/cloudformation/data.py index 92588b024..7ea88cb9e 100644 --- a/deployment/cloudformation/data.py +++ b/deployment/cloudformation/data.py @@ -241,8 +241,8 @@ class DataPlaneGenerator(StackNode): def set_up_stack(self): """Sets up the stack""" - super(DataPlaneGenerator, self).set_up_stack() - self.add_description('Data Plane Stack for Cac') + self.set_version() + self.set_description('Data Plane Stack for Cac') self.rds_stack = RDSFactory() self.rds_stack.populate_template(self) for key in self.parameters: diff --git a/deployment/cloudformation/vpc.py b/deployment/cloudformation/vpc.py index 8c1496c74..fc0cd62bc 100644 --- a/deployment/cloudformation/vpc.py +++ b/deployment/cloudformation/vpc.py @@ -81,7 +81,7 @@ class VPC(StackNode): def set_up_stack(self): """Sets up the stack""" - super(VPC, self).set_up_stack() + self.set_version() """Creates a VPC object. See Class docstring for description of class @@ -92,7 +92,7 @@ def set_up_stack(self): tags (dict): Arbitrary tags to add to all resources that accept tags """ tags = self.get_input('Tags').copy() - self.add_description('VPC Stack for Cac') + self.set_description('VPC Stack for Cac') assert isinstance(tags, dict), 'tags must be a dictionary' diff --git a/python/cac_tripplanner/deployment_requirements.txt b/python/cac_tripplanner/deployment_requirements.txt index add39ebba..f872d100c 100644 --- a/python/cac_tripplanner/deployment_requirements.txt +++ b/python/cac_tripplanner/deployment_requirements.txt @@ -2,4 +2,4 @@ boto==2.49.0 PyYAML==5.3.1 majorkirby==1.0.0 requests==2.24.0 -troposphere==2.6.2 +troposphere==3.2.2 From c2e967d0ef24810cd89bc9f68ce08feb2344d071 Mon Sep 17 00:00:00 2001 From: Derek Dohler Date: Mon, 26 Jun 2023 09:35:31 -0400 Subject: [PATCH 2/3] Fix deployment instructions in README --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 43632e44d..d1bf32b3e 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,18 @@ Note that if there is an existing build Graph.obj in `otp_data`, vagrant provisi Building AMIs ------------------------ +1. Configure an AWS profile with `aws configure --profile gophillygo` if you haven't already 1. Make a production group_vars file (similarly to how is described above with development). Make sure production is set to true, and also specify an app_username, which should be set to: ubuntu 2. If building the `otp` machine, make sure the latest GTFS are in `otp_data`, then build a graph when them in the development environment provisioning. This will result in a new `Graph.obj` file being written to `otp_data`. -3. In the project directory within the app VM, run: `deployment/cac-stack.py create-ami --aws-access-key-id YOUR_ACCESS_KEY --aws-secret-access-key YOUR_SECRET_KEY --aws-role-arn YOUR_ASSUMED_ROLE_ARN` +3. In the project directory within the app VM, run: `AWS_PROFILE=gophillygo deployment/cac-stack.py create-ami` 4. The previous command builds all AMIs. To only build a single AMI, run the same command, but also specify the `--machine-type` parameter, which may be set to one of: `bastion`, `otp`, or `app`. Launching AWS Stacks ------------------------ 1. Copy `deployment/default_template.yaml` to `deployment/default.yaml` and edit variables -2. In the project directory, for a set of `Blue` stacks in the `Production` environment, run: `deployment/cac-stack.py launch-stacks --stack-type prod --stack-color blue --aws-access-key-id YOUR_ACCESS_KEY --aws-secret-access-key YOUR_SECRET_KEY --aws-role-arn YOUR_ASSUMED_ROLE_ARN` +1. Configure an AWS profile with `aws configure --profile gophillygo` if you haven't already +2. In the project directory, for a set of `Blue` stacks in the `Production` environment, run: `AWS_PROFILE=gophillygo deployment/cac-stack.py launch-stacks --stack-color blue --stack-type prod` 3. The previous command will do the following: * Ensure the `VPC` stack is up in Production -- it will be launched if it isn't already running * Ensure the `DataPlane` stack is up in Production -- it will be launched if it isn't already running From 14be417b7964bbfbf0bd6a6f90540daaa5f6fc09 Mon Sep 17 00:00:00 2001 From: Derek Dohler Date: Fri, 7 Jul 2023 09:03:30 -0400 Subject: [PATCH 3/3] Don't install deployment deps in app VM Deployment dependencies should be installed in a separate virtualenv and run on-host so that they're not included in the deployed app server. However, the app server depends on boto, so we need to include it in the regular requirements.txt as well. --- README.md | 6 ++++-- deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml | 3 --- python/cac_tripplanner/requirements.txt | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d1bf32b3e..8ef16f68d 100644 --- a/README.md +++ b/README.md @@ -32,14 +32,16 @@ Building AMIs 1. Configure an AWS profile with `aws configure --profile gophillygo` if you haven't already 1. Make a production group_vars file (similarly to how is described above with development). Make sure production is set to true, and also specify an app_username, which should be set to: ubuntu 2. If building the `otp` machine, make sure the latest GTFS are in `otp_data`, then build a graph when them in the development environment provisioning. This will result in a new `Graph.obj` file being written to `otp_data`. -3. In the project directory within the app VM, run: `AWS_PROFILE=gophillygo deployment/cac-stack.py create-ami` -4. The previous command builds all AMIs. To only build a single AMI, run the same command, but also specify the `--machine-type` parameter, which may be set to one of: `bastion`, `otp`, or `app`. +3. Install the deployment dependencies, ideally in a virtualenv: `python3 -m venv .venv && source .venv/bin/activate && pip install -r python/cac_tripplanner/deployment_requirements.txt` +4. Build AMIs by running (within the virtualenv): `AWS_PROFILE=gophillygo deployment/cac-stack.py create-ami` +5. The previous command builds all AMIs. To only build a single AMI, run the same command, but also specify the `--machine-type` parameter, which may be set to one of: `bastion`, `otp`, or `app`. Launching AWS Stacks ------------------------ 1. Copy `deployment/default_template.yaml` to `deployment/default.yaml` and edit variables 1. Configure an AWS profile with `aws configure --profile gophillygo` if you haven't already +1. Create a virtualenv with the deployment dependencies if you haven't already (see Building AMIs, above). 2. In the project directory, for a set of `Blue` stacks in the `Production` environment, run: `AWS_PROFILE=gophillygo deployment/cac-stack.py launch-stacks --stack-color blue --stack-type prod` 3. The previous command will do the following: * Ensure the `VPC` stack is up in Production -- it will be launched if it isn't already running diff --git a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml index 1f89837f0..089d5c558 100644 --- a/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml +++ b/deployment/ansible/roles/cac-tripplanner.app/tasks/main.yml @@ -12,9 +12,6 @@ - libproj-dev - libjpeg-dev -- name: Install pip packages for deployment - pip: requirements={{ root_app_dir }}/deployment_requirements.txt executable=/usr/bin/pip3 - - name: Install pip packages for app pip: requirements={{ root_app_dir }}/requirements.txt executable=/usr/bin/pip3 diff --git a/python/cac_tripplanner/requirements.txt b/python/cac_tripplanner/requirements.txt index d8c0e4aaf..6c580b483 100644 --- a/python/cac_tripplanner/requirements.txt +++ b/python/cac_tripplanner/requirements.txt @@ -1,5 +1,6 @@ base58==2.0.1 boto3==1.15.9 +boto==2.49.0 Django==3.2.13 django-ckeditor==6.0.0 django-image-cropping==1.5.0