From aea48a458cfb6be5859ab9ebe16d44efff6c3177 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 14 Jan 2021 16:00:10 -0800 Subject: [PATCH 1/6] [catalog] rename env file --- ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml | 3 +-- .../catalog/ckan-app/templates/{gunicorn_env.j2 => app_env.j2} | 0 2 files changed, 1 insertion(+), 2 deletions(-) rename ansible/roles/software/ckan/catalog/ckan-app/templates/{gunicorn_env.j2 => app_env.j2} (100%) diff --git a/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml b/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml index 09c86fb88..13419d7da 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml @@ -20,13 +20,12 @@ - name: Copy app .env template: - src: gunicorn_env.j2 + src: app_env.j2 dest: /etc/ckan/.env mode: 0644 owner: root group: root become: true - when: ckan_uses_gunicorn - name: Install supervisor apt: name=supervisor state=present diff --git a/ansible/roles/software/ckan/catalog/ckan-app/templates/gunicorn_env.j2 b/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 similarity index 100% rename from ansible/roles/software/ckan/catalog/ckan-app/templates/gunicorn_env.j2 rename to ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 From cf585d4d225924c3e7dd0e5dfe2cf840aff8b855 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 14 Jan 2021 16:07:13 -0800 Subject: [PATCH 2/6] [catalog][inventory] dot env config from ansible Specify CKAN application environment variables from ansible configuration instead of template. --- .../software/ckan/catalog/ckan-app/defaults/main.yml | 1 + .../catalog/ckan-app/molecule/default/molecule.yml | 2 ++ .../ckan-app/molecule/default/tests/test_default.py | 10 ++++++++++ .../ckan/catalog/ckan-app/templates/app_env.j2 | 5 ++++- .../roles/software/ckan/inventory/defaults/main.yml | 1 + .../ckan/inventory/molecule/default/molecule.yml | 2 ++ .../inventory/molecule/default/tests/test_default.py | 10 ++++++++++ .../roles/software/ckan/inventory/templates/app_env.j2 | 5 ++++- 8 files changed, 34 insertions(+), 2 deletions(-) diff --git a/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml b/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml index 11ee5c79d..c41e6b7ee 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml @@ -5,6 +5,7 @@ catalog_app_type: web # either web or worker catalog_ckan_access_log: "{{ catalog_log_dir }}/ckan.access.log" catalog_ckan_apache_server_alias: [] catalog_ckan_apache_server_name: ckan +catalog_ckan_envs: "" # One of [default, writeonly, readonly] # default: stand-alone instance, handles both read and write operations. diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml index 58fa89d55..3c63f1150 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml @@ -29,6 +29,8 @@ provisioner: all: catalog_ckan_redis_password: redispass # fake secret for test catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test + catalog_ckan_envs: | + TEST_ENV=1 newrelic_license_key: some-secret # https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9 redis_travis_ci: true diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/tests/test_default.py b/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/tests/test_default.py index 3463514cf..0d624188d 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/tests/test_default.py +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/tests/test_default.py @@ -132,3 +132,13 @@ def test_apache_site(host): 'Expected no rewrite rule for login URLs' assert not f.contains('RewriteCond.*!auth_tkt'), \ 'Expected no rewrite condition for unauthenticated requests' + + +def test_ckan_dot_env(host): + dot_env = host.file('/etc/ckan/.env') + + assert dot_env.exists + assert dot_env.user == 'root' + assert dot_env.group == 'www-data' + assert dot_env.mode == 0o640 + assert dot_env.contains('TEST_ENV=1') diff --git a/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 b/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 index 48d2db827..d673b3861 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 +++ b/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 @@ -1,7 +1,10 @@ +{{ ansible_managed }} NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}" NEW_RELIC_APP_NAME="{{ newrelic_app_name }}" NEW_RELIC_MONITOR_MODE=true NEW_RELIC_LOG=/var/log/new_relic.log NEW_RELIC_LOG_LEVEL=info NEW_RELIC_HOST=gov-collector.newrelic.com -CKAN_INI=/etc/ckan/production.ini \ No newline at end of file +CKAN_INI=/etc/ckan/production.ini + +{{ catalog_ckan_envs }} diff --git a/ansible/roles/software/ckan/inventory/defaults/main.yml b/ansible/roles/software/ckan/inventory/defaults/main.yml index 3a83909c0..95b764b1f 100644 --- a/ansible/roles/software/ckan/inventory/defaults/main.yml +++ b/ansible/roles/software/ckan/inventory/defaults/main.yml @@ -11,6 +11,7 @@ ckan_virtual_env: "{{virtual_env}}" datapusher_virtual_env: /usr/lib/datapusher app_type: inventory +inventory_ckan_envs: "" inventory_ckan_plugins_additional: [] inventory_ckan_plugins_default: - datajson diff --git a/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml b/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml index accefa579..b87de661f 100644 --- a/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml +++ b/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml @@ -32,6 +32,8 @@ provisioner: python_home: /usr inventory_ckan_solr_port: 8983 db_is_setup: false + inventory_ckan_envs: | + TEST_ENV=1 newrelic_license_key: some-secret newrelic_app_name: inventory-molecule newrelic_enabled : false diff --git a/ansible/roles/software/ckan/inventory/molecule/default/tests/test_default.py b/ansible/roles/software/ckan/inventory/molecule/default/tests/test_default.py index 58aacfb5d..7cb22b313 100644 --- a/ansible/roles/software/ckan/inventory/molecule/default/tests/test_default.py +++ b/ansible/roles/software/ckan/inventory/molecule/default/tests/test_default.py @@ -83,3 +83,13 @@ def test_beaker_cache_cleanup(host): def test_ckan_process(host): supervisor_output = host.check_output('supervisorctl status') assert re.search(r'ckan +RUNNING', supervisor_output) + + +def test_ckan_dot_env(host): + dot_env = host.file('/etc/ckan/.env') + + assert dot_env.exists + assert dot_env.user == 'root' + assert dot_env.group == 'www-data' + assert dot_env.mode == 0o640 + assert dot_env.contains('TEST_ENV=1') diff --git a/ansible/roles/software/ckan/inventory/templates/app_env.j2 b/ansible/roles/software/ckan/inventory/templates/app_env.j2 index 408956799..ddfd352e1 100644 --- a/ansible/roles/software/ckan/inventory/templates/app_env.j2 +++ b/ansible/roles/software/ckan/inventory/templates/app_env.j2 @@ -1,3 +1,4 @@ +{{ ansible_managed }} # New Relic NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}" NEW_RELIC_APP_NAME="{{ newrelic_app_name }}" @@ -5,4 +6,6 @@ NEW_RELIC_MONITOR_MODE="{{ newrelic_enabled }}" NEW_RELIC_LOG=/var/log/inventory/new_relic.log NEW_RELIC_LOG_LEVEL=info NEW_RELIC_HOST=gov-collector.newrelic.com -CKAN_INI=/etc/ckan/production.ini \ No newline at end of file +CKAN_INI=/etc/ckan/production.ini + +{{ inventory_ckan_envs }} From 54350ebcbe96710aafadf3ae016282f9a2c29583 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 14 Jan 2021 16:14:40 -0800 Subject: [PATCH 3/6] [catalog][inventory] move NR env variables Move NR env variables out to Ansible configuration instead of template. Now catalog/inventory don't need to know anything about New Relic. --- ansible/group_vars/all/vars.yml | 8 ++++++++ ansible/group_vars/catalog-next/vars.yml | 1 + ansible/group_vars/inventory-next/vars.yml | 2 +- ansible/inventories/staging/group_vars/all/vars.yml | 4 ++++ .../software/ckan/catalog/ckan-app/defaults/main.yml | 1 - .../catalog/ckan-app/molecule/catalog-next/molecule.yml | 1 - .../ckan/catalog/ckan-app/molecule/default/molecule.yml | 1 - .../catalog/ckan-app/molecule/in_service/molecule.yml | 1 - .../ckan/catalog/ckan-app/molecule/readwrite/molecule.yml | 1 - .../ckan/catalog/ckan-app/molecule/saml2/molecule.yml | 1 - .../catalog/ckan-app/molecule/worker-next/molecule.yml | 1 - .../catalog/ckan-app/molecule/worker_main/molecule.yml | 1 - .../roles/software/ckan/catalog/ckan-app/tasks/web.yml | 5 ----- .../software/ckan/catalog/ckan-app/templates/app_env.j2 | 6 ------ .../software/ckan/inventory/molecule/default/molecule.yml | 3 --- .../ckan/inventory/molecule/in_service/molecule.yml | 3 --- .../ckan/inventory/molecule/inventory-next/molecule.yml | 3 --- ansible/roles/software/ckan/inventory/tasks/main.yml | 5 ----- .../roles/software/ckan/inventory/templates/app_env.j2 | 7 ------- 19 files changed, 14 insertions(+), 41 deletions(-) diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index 1852b89aa..153a6a51e 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -137,6 +137,14 @@ jumpbox_ips: # newrelic monitoring +# APM environment variables for applications +newrelic_app_envs: | + NEW_RELIC_APP_NAME="{{ newrelic_app_name }}{% if newrelic_environment != 'production' %} ({{ newrelic_environment }}){% endif %}" + NEW_RELIC_HOST=gov-collector.newrelic.com + NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}" + NEW_RELIC_LOG=/var/log/new_relic.log + NEW_RELIC_LOG_LEVEL=info + NEW_RELIC_MONITOR_MODE={{ newrelic_enabled | ternary('true', 'false') }} newrelic_environment: "{{ datagov_environment | default('unknown') }}" nrinfragent_config: license_key: "{{ newrelic_license_key }}" diff --git a/ansible/group_vars/catalog-next/vars.yml b/ansible/group_vars/catalog-next/vars.yml index 182c87cef..345b6f6c0 100644 --- a/ansible/group_vars/catalog-next/vars.yml +++ b/ansible/group_vars/catalog-next/vars.yml @@ -6,6 +6,7 @@ ckan_catalog_next: true ckan_uses_gunicorn: true ckan_production_ini_template: catalog-next/etc_ckan_production.ini.j2 catalog_ckan_app_version: master +catalog_ckan_envs: "{{ catalog_next_ckan_envs }}" catalog_ckan_saml2_enabled: "{{ catalog_next_ckan_saml2_enabled }}" catalog_ckan_who_ini_path: "{{ catalog_next_ckan_who_ini_path }}" diff --git a/ansible/group_vars/inventory-next/vars.yml b/ansible/group_vars/inventory-next/vars.yml index e551cc0b3..e32321d03 100644 --- a/ansible/group_vars/inventory-next/vars.yml +++ b/ansible/group_vars/inventory-next/vars.yml @@ -1,3 +1,3 @@ --- - +inventory_ckan_envs: "{{ inventory_next_ckan_envs }}" inventory_next: true diff --git a/ansible/inventories/staging/group_vars/all/vars.yml b/ansible/inventories/staging/group_vars/all/vars.yml index e97e56ce1..b6467969f 100644 --- a/ansible/inventories/staging/group_vars/all/vars.yml +++ b/ansible/inventories/staging/group_vars/all/vars.yml @@ -36,6 +36,8 @@ catalog_next_ckan_db_pass: "{{ vault_catalog_next_ckan_db_pass }}" catalog_next_ckan_db_primary_host: "{{ vault_catalog_next_ckan_db_primary_host }}" catalog_next_ckan_db_replica_host: "{{ vault_catalog_next_ckan_db_replica_host }}" catalog_next_ckan_db_user: "{{ vault_catalog_next_ckan_db_user }}" +catalog_next_ckan_envs: | + {{ newrelic_app_envs }} catalog_next_ckan_instance_secret: "{{ vault_catalog_next_ckan_instance_secret }}" catalog_next_ckan_instance_uuid: 1d6ce0c2-1e55-48c5-8d2a-37452ff57989 catalog_next_ckan_postgresql_admin_host: "{{ vault_catalog_next_ckan_postgresql_admin_host }}" @@ -193,6 +195,8 @@ inventory_ckan_solr_port: "8983" inventory_ckan_who_ini_path: "etc_ckan_who.saml2.ini.j2" # Inventory Next +inventory_next_ckan_envs: | + {{ newrelic_app_envs }} inventory_next_ckan_redis_host: "redis1d.dev-ocsit.bsp.gsa.gov" inventory_next_ckan_redis_password: "{{ redis_password }}" inventory_next_ckan_s3_bucket_name: "{{ vault_inventory_next_ckan_s3_bucket_name }}" diff --git a/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml b/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml index c41e6b7ee..64bb68d4c 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/defaults/main.yml @@ -105,4 +105,3 @@ ckan_catalog_next: false catalog_ckan_who_ini_path: etc_ckan_who.default.ini.j2 ckan_production_ini_template: etc_ckan_production.ini.j2 ckan_uses_gunicorn: false -newrelic_app_name: catalog diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/catalog-next/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/catalog-next/molecule.yml index 7f51ecc2d..b4abbf141 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/catalog-next/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/catalog-next/molecule.yml @@ -113,7 +113,6 @@ provisioner: - datagovtheme catalog_ckan_plugins_additional: - saml2auth - newrelic_license_key: some-secret lint: | set -e ansible-lint diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml index 3c63f1150..e63c87558 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/default/molecule.yml @@ -31,7 +31,6 @@ provisioner: catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test catalog_ckan_envs: | TEST_ENV=1 - newrelic_license_key: some-secret # https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9 redis_travis_ci: true v2: diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/in_service/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/in_service/molecule.yml index 546401a31..b0b5e0b18 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/in_service/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/in_service/molecule.yml @@ -32,7 +32,6 @@ provisioner: all: catalog_ckan_redis_password: redispass # fake secret for test catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test - newrelic_license_key: some-secret v2: app_repo_branch: bionic lint: | diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/readwrite/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/readwrite/molecule.yml index 6a2f66b8a..16e27e6df 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/readwrite/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/readwrite/molecule.yml @@ -29,7 +29,6 @@ provisioner: all: catalog_ckan_redis_password: redispass # fake secret for test catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 - newrelic_license_key: some-secret # https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9 redis_travis_ci: true v2: diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/saml2/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/saml2/molecule.yml index 96f893293..7d5bdf4b9 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/saml2/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/saml2/molecule.yml @@ -32,7 +32,6 @@ provisioner: all: catalog_ckan_redis_password: redispass # fake secret for test catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test - newrelic_license_key: some-secret v2: app_repo_branch: bionic lint: | diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker-next/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker-next/molecule.yml index 36b45105d..1983d54c5 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker-next/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker-next/molecule.yml @@ -47,7 +47,6 @@ provisioner: - geodatagov_waf_harvester - spatial_query - datagovtheme - newrelic_license_key: some-secret # https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9 redis_travis_ci: true lint: | diff --git a/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker_main/molecule.yml b/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker_main/molecule.yml index abcbfcf38..3bf7c385c 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker_main/molecule.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/molecule/worker_main/molecule.yml @@ -33,7 +33,6 @@ provisioner: catalog_ckan_db_user: ckan catalog_ckan_redis_password: redispass # fake secret for test catalog_ckan_who_ini_secret: e45cfed3-40f1-41c0-8e92-77eda7ddd9f3 # Fake secret for test - newrelic_license_key: some-secret # https://github.com/DavidWittman/ansible-redis/blob/21b0b6f9030275a2586baf591f322ce3348b2b2d/tasks/install.yml#L9 redis_travis_ci: true v2: diff --git a/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml b/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml index 13419d7da..97532f1d6 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml @@ -1,9 +1,4 @@ --- -- name: Assert newrelic_license_key is set - assert: - that: newrelic_license_key is defined - fail_msg: newrelic_license_key is required but it is not set - - name: Install WSGI app copy: src=etc_ckan_apache.wsgi dest=/etc/ckan/apache.wsgi mode=0644 owner=root group=www-data notify: reload apache2 diff --git a/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 b/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 index d673b3861..9ec73aea2 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 +++ b/ansible/roles/software/ckan/catalog/ckan-app/templates/app_env.j2 @@ -1,10 +1,4 @@ {{ ansible_managed }} -NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}" -NEW_RELIC_APP_NAME="{{ newrelic_app_name }}" -NEW_RELIC_MONITOR_MODE=true -NEW_RELIC_LOG=/var/log/new_relic.log -NEW_RELIC_LOG_LEVEL=info -NEW_RELIC_HOST=gov-collector.newrelic.com CKAN_INI=/etc/ckan/production.ini {{ catalog_ckan_envs }} diff --git a/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml b/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml index b87de661f..472aacddb 100644 --- a/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml +++ b/ansible/roles/software/ckan/inventory/molecule/default/molecule.yml @@ -34,9 +34,6 @@ provisioner: db_is_setup: false inventory_ckan_envs: | TEST_ENV=1 - newrelic_license_key: some-secret - newrelic_app_name: inventory-molecule - newrelic_enabled : false bionic: inventory_app_repo_branch: inventory_ckan_2.8 inventory_next: true diff --git a/ansible/roles/software/ckan/inventory/molecule/in_service/molecule.yml b/ansible/roles/software/ckan/inventory/molecule/in_service/molecule.yml index bf9003a1a..f549e5b0c 100644 --- a/ansible/roles/software/ckan/inventory/molecule/in_service/molecule.yml +++ b/ansible/roles/software/ckan/inventory/molecule/in_service/molecule.yml @@ -33,9 +33,6 @@ provisioner: python_home: /usr inventory_ckan_solr_port: 8983 db_is_setup: false - newrelic_license_key: some-secret - newrelic_app_name: inventory-molecule - newrelic_enabled: false bionic: inventory_app_repo_branch: inventory_ckan_2.8 datapusher_build_pkg_branch: datagov/inventory-next diff --git a/ansible/roles/software/ckan/inventory/molecule/inventory-next/molecule.yml b/ansible/roles/software/ckan/inventory/molecule/inventory-next/molecule.yml index 039ee89ab..cd8ab5b8a 100644 --- a/ansible/roles/software/ckan/inventory/molecule/inventory-next/molecule.yml +++ b/ansible/roles/software/ckan/inventory/molecule/inventory-next/molecule.yml @@ -31,9 +31,6 @@ provisioner: inventory_app_repo_branch: inventory_ckan_2.8 inventory_next: true datapusher_build_pkg_branch: datagov/inventory-next - newrelic_license_key: some-secret - newrelic_app_name: inventory-molecule - newrelic_enabled: true inventory_ckan_saml2_entity_id: urn:gov:gsa:SAML:2.0.profiles:sp:sso:gsa:datagov-sandbox-inventory inventory_next_ckan_who_ini_secret: some-secret scenario: diff --git a/ansible/roles/software/ckan/inventory/tasks/main.yml b/ansible/roles/software/ckan/inventory/tasks/main.yml index c8cd8f11c..510e9e60c 100644 --- a/ansible/roles/software/ckan/inventory/tasks/main.yml +++ b/ansible/roles/software/ckan/inventory/tasks/main.yml @@ -9,11 +9,6 @@ - cron - supervisor -- name: Assert newrelic_license_key is set - assert: - that: newrelic_license_key is defined - fail_msg: newrelic_license_key is required but it is not set - - name: Create log directory file: path={{ inventory_log_dir }} state=directory owner=root group=www-data mode=0750 diff --git a/ansible/roles/software/ckan/inventory/templates/app_env.j2 b/ansible/roles/software/ckan/inventory/templates/app_env.j2 index ddfd352e1..389246ecf 100644 --- a/ansible/roles/software/ckan/inventory/templates/app_env.j2 +++ b/ansible/roles/software/ckan/inventory/templates/app_env.j2 @@ -1,11 +1,4 @@ {{ ansible_managed }} -# New Relic -NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}" -NEW_RELIC_APP_NAME="{{ newrelic_app_name }}" -NEW_RELIC_MONITOR_MODE="{{ newrelic_enabled }}" -NEW_RELIC_LOG=/var/log/inventory/new_relic.log -NEW_RELIC_LOG_LEVEL=info -NEW_RELIC_HOST=gov-collector.newrelic.com CKAN_INI=/etc/ckan/production.ini {{ inventory_ckan_envs }} From 8ec59aeb2aabf4b57e68e45324e8c84df3ec7c1b Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 14 Jan 2021 16:45:18 -0800 Subject: [PATCH 4/6] [catalog][inventory] add variable for other envs Missing some variables, make sure to set this for all environments. --- ansible/inventories/production/group_vars/all/vars.yml | 4 ++++ ansible/inventories/sandbox/group_vars/all/vars.yml | 2 ++ 2 files changed, 6 insertions(+) diff --git a/ansible/inventories/production/group_vars/all/vars.yml b/ansible/inventories/production/group_vars/all/vars.yml index bc839315b..95cb443f9 100644 --- a/ansible/inventories/production/group_vars/all/vars.yml +++ b/ansible/inventories/production/group_vars/all/vars.yml @@ -34,6 +34,8 @@ catalog_next_ckan_db_primary_host: "{{ vault_catalog_next_ckan_db_primary_host } catalog_next_ckan_db_replica_a_host: "{{ vault_catalog_next_ckan_db_replica_a_host }}" catalog_next_ckan_db_replica_b_host: "{{ vault_catalog_next_ckan_db_replica_b_host }}" catalog_next_ckan_db_user: "{{ vault_catalog_next_ckan_db_user }}" +catalog_next_ckan_envs: | + {{ newrelic_app_envs }} catalog_next_ckan_instance_secret: "{{ vault_catalog_next_ckan_instance_secret }}" catalog_next_ckan_instance_uuid: 5ab5625f-4ec5-435b-a725-55eaa36d264b catalog_next_ckan_postgresql_admin_host: "{{ vault_catalog_next_ckan_postgresql_admin_host }}" @@ -203,6 +205,8 @@ inventory_ckan_solr_port: "8983" inventory_ckan_who_ini_path: "etc_ckan_who.saml2.ini.j2" # Inventory Next +inventory_next_ckan_envs: | + {{ newrelic_app_envs }} inventory_next_ckan_redis_host: "redis1p.prod-ocsit.bsp.gsa.gov" inventory_next_ckan_redis_password: "{{ redis_password }}" inventory_next_ckan_s3_bucket_name: "{{ vault_inventory_next_ckan_s3_bucket_name }}" diff --git a/ansible/inventories/sandbox/group_vars/all/vars.yml b/ansible/inventories/sandbox/group_vars/all/vars.yml index e7c6e9617..14470cd0d 100644 --- a/ansible/inventories/sandbox/group_vars/all/vars.yml +++ b/ansible/inventories/sandbox/group_vars/all/vars.yml @@ -30,6 +30,7 @@ catalog_db_user: "{{ catalog_ckan_db_user }}" catalog_next_ckan_db_name: ckan catalog_next_ckan_db_pass: "{{ vault_catalog_next_ckan_db_pass }}" catalog_next_ckan_db_user: ckan +catalog_next_ckan_envs: "" catalog_next_ckan_fgdc2iso_host: catalog-next-fgdc2iso1tf.internal.sandbox.datagov.us catalog_next_ckan_redis_host: master.rep-sandbox-catalog-next.5kspe7.use1.cache.amazonaws.com catalog_next_ckan_redis_password: "{{ vault_catalog_next_ckan_redis_password }}" @@ -112,6 +113,7 @@ inventory_postgresql_login_password: "{{ vault_inventory_postgresql_login_passwo inventory_ckan_solr_port: "{{ solr_port }}" # Inventory Next +inventory_next_ckan_envs: "" inventory_next_ckan_s3_bucket_name: "{{ vault_inventory_next_ckan_s3_bucket_name }}" inventory_next_ckan_s3_bucket_prefix: "{{ vault_inventory_next_ckan_s3_bucket_prefix }}" inventory_next_ckan_instance_secret: "{{ vault_inventory_next_ckan_instance_secret }}" From 8de7d054c3cbbf4ce7b101b66904ee5e0485cca5 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 14 Jan 2021 17:01:21 -0800 Subject: [PATCH 5/6] [newrelic] refactor newrelic_enabled Make sure to enable New Relic in production and staging. Set a default in case it's unset. --- ansible/group_vars/all/vars.yml | 2 +- ansible/inventories/production/group_vars/all/vars.yml | 1 + ansible/inventories/sandbox/group_vars/inventory-next/vars.yml | 3 +-- ansible/inventories/staging/group_vars/all/vars.yml | 1 + ansible/inventories/staging/group_vars/inventory-next/vars.yml | 3 +-- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index 153a6a51e..5464c2f37 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -144,7 +144,7 @@ newrelic_app_envs: | NEW_RELIC_LICENSE_KEY="{{ newrelic_license_key }}" NEW_RELIC_LOG=/var/log/new_relic.log NEW_RELIC_LOG_LEVEL=info - NEW_RELIC_MONITOR_MODE={{ newrelic_enabled | ternary('true', 'false') }} + NEW_RELIC_MONITOR_MODE={{ newrelic_enabled | default(False) | ternary('true', 'false') }} newrelic_environment: "{{ datagov_environment | default('unknown') }}" nrinfragent_config: license_key: "{{ newrelic_license_key }}" diff --git a/ansible/inventories/production/group_vars/all/vars.yml b/ansible/inventories/production/group_vars/all/vars.yml index 95cb443f9..1e34f8ef1 100644 --- a/ansible/inventories/production/group_vars/all/vars.yml +++ b/ansible/inventories/production/group_vars/all/vars.yml @@ -229,6 +229,7 @@ inventory_next_postgresql_db_name: "{{ vault_inventory_next_postgresql_db_name } inventory_next_datapusher_db_name: "{{ vault_inventory_next_datapusher_db_name }}" # New Relic +newrelic_enabled: true newrelic_license_key: "{{ vault_newrelic_license_key }}" diff --git a/ansible/inventories/sandbox/group_vars/inventory-next/vars.yml b/ansible/inventories/sandbox/group_vars/inventory-next/vars.yml index 6b9813933..6d64c6555 100644 --- a/ansible/inventories/sandbox/group_vars/inventory-next/vars.yml +++ b/ansible/inventories/sandbox/group_vars/inventory-next/vars.yml @@ -30,8 +30,7 @@ inventory_ckan_bucket_prefix: "{{ inventory_next_ckan_s3_bucket_prefix }}" ckan_site_domain: "{{ inventory_next_ckan_service_url }}" inventory_app_repo_branch: inventory_ckan_2.8 -newrelic_app_name: inventory-next-sandbox -newrelic_enabled: false +newrelic_app_name: inventory-next inventory_ckan_plugins_additional: [saml2auth s3filestore] diff --git a/ansible/inventories/staging/group_vars/all/vars.yml b/ansible/inventories/staging/group_vars/all/vars.yml index b6467969f..b21314a8e 100644 --- a/ansible/inventories/staging/group_vars/all/vars.yml +++ b/ansible/inventories/staging/group_vars/all/vars.yml @@ -219,6 +219,7 @@ inventory_next_postgresql_db_name: "{{ vault_inventory_next_postgresql_db_name } inventory_next_datapusher_db_name: "{{ vault_inventory_next_datapusher_db_name }}" # New Relic +newrelic_enabled: true newrelic_license_key: "{{ vault_newrelic_license_key }}" diff --git a/ansible/inventories/staging/group_vars/inventory-next/vars.yml b/ansible/inventories/staging/group_vars/inventory-next/vars.yml index d0ab896c2..1ee28e501 100644 --- a/ansible/inventories/staging/group_vars/inventory-next/vars.yml +++ b/ansible/inventories/staging/group_vars/inventory-next/vars.yml @@ -32,8 +32,7 @@ inventory_ckan_bucket_prefix: "{{ inventory_next_ckan_s3_bucket_prefix }}" ckan_site_domain: "{{ inventory_next_ckan_service_url }}" inventory_app_repo_branch: inventory_ckan_2.8 -newrelic_app_name: inventory-next-staging -newrelic_enabled: true +newrelic_app_name: inventory-next inventory_ckan_plugins_additional: [saml2auth s3filestore] From 4b98ed277600c248e10b624bdf318c3789c89100 Mon Sep 17 00:00:00 2001 From: Aaron D Borden Date: Thu, 14 Jan 2021 17:33:48 -0800 Subject: [PATCH 6/6] Fix tests dot env can contain secrets, limit its access. --- ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml | 4 ++-- ansible/roles/software/ckan/inventory/tasks/main.yml | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml b/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml index 97532f1d6..fd5f7f685 100644 --- a/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml +++ b/ansible/roles/software/ckan/catalog/ckan-app/tasks/web.yml @@ -17,9 +17,9 @@ template: src: app_env.j2 dest: /etc/ckan/.env - mode: 0644 + mode: 0640 owner: root - group: root + group: www-data become: true - name: Install supervisor diff --git a/ansible/roles/software/ckan/inventory/tasks/main.yml b/ansible/roles/software/ckan/inventory/tasks/main.yml index 510e9e60c..48e97ae06 100644 --- a/ansible/roles/software/ckan/inventory/tasks/main.yml +++ b/ansible/roles/software/ckan/inventory/tasks/main.yml @@ -116,11 +116,10 @@ template: src: app_env.j2 dest: /etc/ckan/.env - mode: 0644 + mode: 0640 owner: root - group: root + group: www-data become: true - when: inventory_next notify: - restart ckan