Skip to content

Commit

Permalink
Merge pull request #2651 from GSA/feature/app-env
Browse files Browse the repository at this point in the history
[catalog][inventory] fix NR reporting
  • Loading branch information
adborden authored Feb 1, 2021
2 parents 8e160a4 + 4b98ed2 commit dcba2e6
Show file tree
Hide file tree
Showing 27 changed files with 63 additions and 54 deletions.
8 changes: 8 additions & 0 deletions ansible/group_vars/all/vars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,14 @@ jumpbox_ips:


# newrelic monitoring
# APM environment variables for applications
newrelic_app_envs: |
NEW_RELIC_APP_NAME="{{ newrelic_app_name }}{% if newrelic_environment != 'production' %} ({{ newrelic_environment }}){% endif %}"
NEW_RELIC_HOST=gov-collector.newrelic.com
NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}"
NEW_RELIC_LOG=/var/log/new_relic.log
NEW_RELIC_LOG_LEVEL=info
NEW_RELIC_MONITOR_MODE={{ newrelic_enabled | default(False) | ternary('true', 'false') }}
newrelic_environment: "{{ datagov_environment | default('unknown') }}"
nrinfragent_config:
license_key: "{{ newrelic_license_key }}"
Expand Down
1 change: 1 addition & 0 deletions ansible/group_vars/catalog-next/vars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ ckan_catalog_next: true
ckan_uses_gunicorn: true
ckan_production_ini_template: catalog-next/etc_ckan_production.ini.j2
catalog_ckan_app_version: master
catalog_ckan_envs: "{{ catalog_next_ckan_envs }}"
catalog_ckan_saml2_enabled: "{{ catalog_next_ckan_saml2_enabled }}"
catalog_ckan_who_ini_path: "{{ catalog_next_ckan_who_ini_path }}"

Expand Down
2 changes: 1 addition & 1 deletion ansible/group_vars/inventory-next/vars.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
---

inventory_ckan_envs: "{{ inventory_next_ckan_envs }}"
inventory_next: true
5 changes: 5 additions & 0 deletions ansible/inventories/production/group_vars/all/vars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ catalog_next_ckan_db_primary_host: "{{ vault_catalog_next_ckan_db_primary_host }
catalog_next_ckan_db_replica_a_host: "{{ vault_catalog_next_ckan_db_replica_a_host }}"
catalog_next_ckan_db_replica_b_host: "{{ vault_catalog_next_ckan_db_replica_b_host }}"
catalog_next_ckan_db_user: "{{ vault_catalog_next_ckan_db_user }}"
catalog_next_ckan_envs: |
{{ newrelic_app_envs }}
catalog_next_ckan_instance_secret: "{{ vault_catalog_next_ckan_instance_secret }}"
catalog_next_ckan_instance_uuid: 5ab5625f-4ec5-435b-a725-55eaa36d264b
catalog_next_ckan_postgresql_admin_host: "{{ vault_catalog_next_ckan_postgresql_admin_host }}"
Expand Down Expand Up @@ -203,6 +205,8 @@ inventory_ckan_solr_port: "8983"
inventory_ckan_who_ini_path: "etc_ckan_who.saml2.ini.j2"

# Inventory Next
inventory_next_ckan_envs: |
{{ newrelic_app_envs }}
inventory_next_ckan_redis_host: "redis1p.prod-ocsit.bsp.gsa.gov"
inventory_next_ckan_redis_password: "{{ redis_password }}"
inventory_next_ckan_s3_bucket_name: "{{ vault_inventory_next_ckan_s3_bucket_name }}"
Expand All @@ -225,6 +229,7 @@ inventory_next_postgresql_db_name: "{{ vault_inventory_next_postgresql_db_name }
inventory_next_datapusher_db_name: "{{ vault_inventory_next_datapusher_db_name }}"

# New Relic
newrelic_enabled: true
newrelic_license_key: "{{ vault_newrelic_license_key }}"


Expand Down
2 changes: 2 additions & 0 deletions ansible/inventories/sandbox/group_vars/all/vars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ catalog_db_user: "{{ catalog_ckan_db_user }}"
catalog_next_ckan_db_name: ckan
catalog_next_ckan_db_pass: "{{ vault_catalog_next_ckan_db_pass }}"
catalog_next_ckan_db_user: ckan
catalog_next_ckan_envs: ""
catalog_next_ckan_fgdc2iso_host: catalog-next-fgdc2iso1tf.internal.sandbox.datagov.us
catalog_next_ckan_redis_host: master.rep-sandbox-catalog-next.5kspe7.use1.cache.amazonaws.com
catalog_next_ckan_redis_password: "{{ vault_catalog_next_ckan_redis_password }}"
Expand Down Expand Up @@ -112,6 +113,7 @@ inventory_postgresql_login_password: "{{ vault_inventory_postgresql_login_passwo
inventory_ckan_solr_port: "{{ solr_port }}"

# Inventory Next
inventory_next_ckan_envs: ""
inventory_next_ckan_s3_bucket_name: "{{ vault_inventory_next_ckan_s3_bucket_name }}"
inventory_next_ckan_s3_bucket_prefix: "{{ vault_inventory_next_ckan_s3_bucket_prefix }}"
inventory_next_ckan_instance_secret: "{{ vault_inventory_next_ckan_instance_secret }}"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,7 @@ inventory_ckan_bucket_prefix: "{{ inventory_next_ckan_s3_bucket_prefix }}"
ckan_site_domain: "{{ inventory_next_ckan_service_url }}"

inventory_app_repo_branch: inventory_ckan_2.8
newrelic_app_name: inventory-next-sandbox
newrelic_enabled: false
newrelic_app_name: inventory-next

inventory_ckan_plugins_additional: [saml2auth s3filestore]

Expand Down
5 changes: 5 additions & 0 deletions ansible/inventories/staging/group_vars/all/vars.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ catalog_next_ckan_db_pass: "{{ vault_catalog_next_ckan_db_pass }}"
catalog_next_ckan_db_primary_host: "{{ vault_catalog_next_ckan_db_primary_host }}"
catalog_next_ckan_db_replica_host: "{{ vault_catalog_next_ckan_db_replica_host }}"
catalog_next_ckan_db_user: "{{ vault_catalog_next_ckan_db_user }}"
catalog_next_ckan_envs: |
{{ newrelic_app_envs }}
catalog_next_ckan_instance_secret: "{{ vault_catalog_next_ckan_instance_secret }}"
catalog_next_ckan_instance_uuid: 1d6ce0c2-1e55-48c5-8d2a-37452ff57989
catalog_next_ckan_postgresql_admin_host: "{{ vault_catalog_next_ckan_postgresql_admin_host }}"
Expand Down Expand Up @@ -193,6 +195,8 @@ inventory_ckan_solr_port: "8983"
inventory_ckan_who_ini_path: "etc_ckan_who.saml2.ini.j2"

# Inventory Next
inventory_next_ckan_envs: |
{{ newrelic_app_envs }}
inventory_next_ckan_redis_host: "redis1d.dev-ocsit.bsp.gsa.gov"
inventory_next_ckan_redis_password: "{{ redis_password }}"
inventory_next_ckan_s3_bucket_name: "{{ vault_inventory_next_ckan_s3_bucket_name }}"
Expand All @@ -215,6 +219,7 @@ inventory_next_postgresql_db_name: "{{ vault_inventory_next_postgresql_db_name }
inventory_next_datapusher_db_name: "{{ vault_inventory_next_datapusher_db_name }}"

# New Relic
newrelic_enabled: true
newrelic_license_key: "{{ vault_newrelic_license_key }}"


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,7 @@ inventory_ckan_bucket_prefix: "{{ inventory_next_ckan_s3_bucket_prefix }}"
ckan_site_domain: "{{ inventory_next_ckan_service_url }}"

inventory_app_repo_branch: inventory_ckan_2.8
newrelic_app_name: inventory-next-staging
newrelic_enabled: true
newrelic_app_name: inventory-next

inventory_ckan_plugins_additional: [saml2auth s3filestore]

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ catalog_app_type: web # either web or worker
catalog_ckan_access_log: "{{ catalog_log_dir }}/ckan.access.log"
catalog_ckan_apache_server_alias: []
catalog_ckan_apache_server_name: ckan
catalog_ckan_envs: ""

# One of [default, writeonly, readonly]
# default: stand-alone instance, handles both read and write operations.
Expand Down Expand Up @@ -104,4 +105,3 @@ ckan_catalog_next: false
catalog_ckan_who_ini_path: etc_ckan_who.default.ini.j2
ckan_production_ini_template: etc_ckan_production.ini.j2
ckan_uses_gunicorn: false
newrelic_app_name: catalog
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,6 @@ provisioner:
- datagovtheme
catalog_ckan_plugins_additional:
- saml2auth
newrelic_license_key: some-secret
lint: |
set -e
ansible-lint
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@ provisioner:
all:
catalog_ckan_redis_password: redispass # fake secret for test
catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test
newrelic_license_key: some-secret
catalog_ckan_envs: |
TEST_ENV=1
# https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9
redis_travis_ci: true
v2:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,3 +132,13 @@ def test_apache_site(host):
'Expected no rewrite rule for login URLs'
assert not f.contains('RewriteCond.*!auth_tkt'), \
'Expected no rewrite condition for unauthenticated requests'


def test_ckan_dot_env(host):
dot_env = host.file('/etc/ckan/.env')

assert dot_env.exists
assert dot_env.user == 'root'
assert dot_env.group == 'www-data'
assert dot_env.mode == 0o640
assert dot_env.contains('TEST_ENV=1')
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ provisioner:
all:
catalog_ckan_redis_password: redispass # fake secret for test
catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test
newrelic_license_key: some-secret
v2:
app_repo_branch: bionic
lint: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ provisioner:
all:
catalog_ckan_redis_password: redispass # fake secret for test
catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3
newrelic_license_key: some-secret
# https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9
redis_travis_ci: true
v2:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ provisioner:
all:
catalog_ckan_redis_password: redispass # fake secret for test
catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test
newrelic_license_key: some-secret
v2:
app_repo_branch: bionic
lint: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ provisioner:
- geodatagov_waf_harvester
- spatial_query
- datagovtheme
newrelic_license_key: some-secret
# https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9
redis_travis_ci: true
lint: |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ provisioner:
catalog_ckan_db_user: ckan
catalog_ckan_redis_password: redispass # fake secret for test
catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test
newrelic_license_key: some-secret
# https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9
redis_travis_ci: true
v2:
Expand Down
12 changes: 3 additions & 9 deletions ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml
Original file line number Diff line number Diff line change
@@ -1,9 +1,4 @@
---
- name: Assert newrelic_license_key is set
assert:
that: newrelic_license_key is defined
fail_msg: newrelic_license_key is required but it is not set

- name: Install WSGI app
copy: src=etc_ckan_apache.wsgi dest=/etc/ckan/apache.wsgi mode=0644 owner=root group=www-data
notify: reload apache2
Expand All @@ -20,13 +15,12 @@

- name: Copy app .env
template:
src: gunicorn_env.j2
src: app_env.j2
dest: /etc/ckan/.env
mode: 0644
mode: 0640
owner: root
group: root
group: www-data
become: true
when: ckan_uses_gunicorn

- name: Install supervisor
apt: name=supervisor state=present
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{{ ansible_managed }}
CKAN_INI=/etc/ckan/production.ini

{{ catalog_ckan_envs }}

This file was deleted.

1 change: 1 addition & 0 deletions ansible/roles/software/ckan/inventory/defaults/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ ckan_virtual_env: "{{virtual_env}}"
datapusher_virtual_env: /usr/lib/datapusher
app_type: inventory

inventory_ckan_envs: ""
inventory_ckan_plugins_additional: []
inventory_ckan_plugins_default:
- datajson
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,8 @@ provisioner:
python_home: /usr
inventory_ckan_solr_port: 8983
db_is_setup: false
newrelic_license_key: some-secret
newrelic_app_name: inventory-molecule
newrelic_enabled : false
inventory_ckan_envs: |
TEST_ENV=1
bionic:
inventory_app_repo_branch: inventory_ckan_2.8
inventory_next: true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,3 +83,13 @@ def test_beaker_cache_cleanup(host):
def test_ckan_process(host):
supervisor_output = host.check_output('supervisorctl status')
assert re.search(r'ckan +RUNNING', supervisor_output)


def test_ckan_dot_env(host):
dot_env = host.file('/etc/ckan/.env')

assert dot_env.exists
assert dot_env.user == 'root'
assert dot_env.group == 'www-data'
assert dot_env.mode == 0o640
assert dot_env.contains('TEST_ENV=1')
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,6 @@ provisioner:
python_home: /usr
inventory_ckan_solr_port: 8983
db_is_setup: false
newrelic_license_key: some-secret
newrelic_app_name: inventory-molecule
newrelic_enabled: false
bionic:
inventory_app_repo_branch: inventory_ckan_2.8
datapusher_build_pkg_branch: datagov/inventory-next
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,6 @@ provisioner:
inventory_app_repo_branch: inventory_ckan_2.8
inventory_next: true
datapusher_build_pkg_branch: datagov/inventory-next
newrelic_license_key: some-secret
newrelic_app_name: inventory-molecule
newrelic_enabled: true
inventory_ckan_saml2_entity_id: urn:gov:gsa:SAML:2.0.profiles:sp:sso:gsa:datagov-sandbox-inventory
inventory_next_ckan_who_ini_secret: some-secret
scenario:
Expand Down
10 changes: 2 additions & 8 deletions ansible/roles/software/ckan/inventory/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@
- cron
- supervisor

- name: Assert newrelic_license_key is set
assert:
that: newrelic_license_key is defined
fail_msg: newrelic_license_key is required but it is not set

- name: Create log directory
file: path={{ inventory_log_dir }} state=directory owner=root group=www-data mode=0750

Expand Down Expand Up @@ -121,11 +116,10 @@
template:
src: app_env.j2
dest: /etc/ckan/.env
mode: 0644
mode: 0640
owner: root
group: root
group: www-data
become: true
when: inventory_next
notify:
- restart ckan

Expand Down
12 changes: 4 additions & 8 deletions ansible/roles/software/ckan/inventory/templates/app_env.j2
Original file line number Diff line number Diff line change
@@ -1,8 +1,4 @@
# New Relic
NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}"
NEW_RELIC_APP_NAME="{{ newrelic_app_name }}"
NEW_RELIC_MONITOR_MODE="{{ newrelic_enabled }}"
NEW_RELIC_LOG=/var/log/inventory/new_relic.log
NEW_RELIC_LOG_LEVEL=info
NEW_RELIC_HOST=gov-collector.newrelic.com
CKAN_INI=/etc/ckan/production.ini
{{ ansible_managed }}
CKAN_INI=/etc/ckan/production.ini

{{ inventory_ckan_envs }}

0 comments on commit dcba2e6

Please sign in to comment.