From b822d681f936589dcd7741426479ad0b7c8bc95c Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Fri, 8 Jul 2016 15:12:14 -0500 Subject: [PATCH 1/5] Change hadoop install dir permissions to 755 --- roles/hadoop/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/hadoop/tasks/main.yml b/roles/hadoop/tasks/main.yml index a320925..bcafe2b 100644 --- a/roles/hadoop/tasks/main.yml +++ b/roles/hadoop/tasks/main.yml @@ -17,7 +17,7 @@ file: path: "{{ item }}" state: directory - mode: 0754 + mode: 0755 owner: "{{ hdfs_user }}" group: "{{ hdfs_group }}" with_items: From 5eadb56ae446b2d99022c04e1a9208c621102b93 Mon Sep 17 00:00:00 2001 From: David Cavitt Date: Fri, 15 Jul 2016 12:56:31 -0400 Subject: [PATCH 2/5] fixed kafka permissions bug. It now sets the correct permissions for the /data directories as well as the /opt directories. --- group_vars/kafka-nodes | 10 +++++++++- host_vars/vagrant-ist-01/vars | 4 ++++ roles/kafka/defaults/main.yml | 2 ++ roles/kafka/tasks/main.yml | 27 ++++++++++++++++----------- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/group_vars/kafka-nodes b/group_vars/kafka-nodes index d5bc9f7..f28262e 100644 --- a/group_vars/kafka-nodes +++ b/group_vars/kafka-nodes @@ -12,9 +12,17 @@ kafka_data_log_dir: - '/data/5/kafka/topic-logs' - '/data/6/kafka/topic-logs' +kafka_data_dir: + - '/data/1/kafka' + - '/data/2/kafka' + - '/data/3/kafka' + - '/data/4/kafka' + - '/data/5/kafka' + - '/data/6/kafka' + # According to Apache Docs, num io threads should be at least equal number of disks. kafka_num_io_threads: 6 kafka_heap: 4G -kafka_log_retention_hours: 24 \ No newline at end of file +kafka_log_retention_hours: 24 diff --git a/host_vars/vagrant-ist-01/vars b/host_vars/vagrant-ist-01/vars index b13da24..c624fde 100644 --- a/host_vars/vagrant-ist-01/vars +++ b/host_vars/vagrant-ist-01/vars @@ -37,6 +37,10 @@ pulse_url_host: "{{ inventory_hostname }}" pulse_listen_port: 443 kafka_heap: 1G +kafka_data_log_dir: + - '/data/kafka/topic-logs' +kafka_data_dir: + - '/data/kafka' sc_crawling_interface: eth0 diff --git a/roles/kafka/defaults/main.yml b/roles/kafka/defaults/main.yml index 7eff3fc..f455b74 100644 --- a/roles/kafka/defaults/main.yml +++ b/roles/kafka/defaults/main.yml @@ -9,6 +9,8 @@ kafka_log_dir: /var/log/kafka kafka_log_rolling_retention_count: 10 kafka_log_rolling_max_size: 25MB +kafka_data_dir: + - "/opt/kafka" kafka_data_log_dir: - "/opt/kafka/topic-logs" diff --git a/roles/kafka/tasks/main.yml b/roles/kafka/tasks/main.yml index 8cc52a2..36dc2a5 100644 --- a/roles/kafka/tasks/main.yml +++ b/roles/kafka/tasks/main.yml @@ -1,15 +1,5 @@ --- -- name: create kafka directories - file: - path={{ item }} - state=directory - mode=0744 - with_items: - - "{{ kafka_install_dir }}" - - "{{ kafka_data_log_dir }}" - tags: kafka - - name: Configuring group group: name: "{{ kafka_group }}" @@ -23,6 +13,17 @@ createhome=yes tags: kafka +- name: create kafka directories + file: + path={{ item }} + state=directory + mode=0744 + with_items: + - "{{ kafka_install_dir }}" + - "{{ kafka_data_log_dir }}" + - "{{ kafka_log_dir }}" + tags: kafka + - name: check for existing install stat: path={{ kafka_install_dir }}/kafka_{{ kafka_scala_version }}-{{ kafka_version }} register: kafka @@ -103,11 +104,15 @@ - name: set owner and group of kafka directory and files file: - path={{ kafka_install_dir }} + path={{ item }} state=directory recurse=yes owner={{ kafka_user }} group={{ kafka_group }} + with_items: + - "{{ kafka_install_dir }}" + - "{{ kafka_data_dir }}" + - "{{ kafka_log_dir }}" tags: kafka # - cron: name="clear old kafka app logs" job="find /opt/kafka/default/logs -mtime +7 -exec rm {} \; > /dev/null" minute="0" From 7f6fb6df323f8c6a106f272365301b3a60c21529 Mon Sep 17 00:00:00 2001 From: David Cavitt Date: Fri, 15 Jul 2016 13:28:48 -0400 Subject: [PATCH 3/5] fix for kafka permissions bug. All kafka files are now owned and operated by kafka user and group. --- roles/kafka/defaults/main.yml | 4 ++- roles/kafka/tasks/main.yml | 27 +++++++++++++++++++ .../kafka/templates/kafka-supervisord.conf.j2 | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/roles/kafka/defaults/main.yml b/roles/kafka/defaults/main.yml index d410fec..0f6d0ca 100644 --- a/roles/kafka/defaults/main.yml +++ b/roles/kafka/defaults/main.yml @@ -9,6 +9,8 @@ kafka_log_dir: /var/log/kafka kafka_log_rolling_retention_count: 10 kafka_log_rolling_max_size: 25MB +kafka_data_dir: + - "/opt/kafka" kafka_data_log_dir: - "/opt/kafka/topic-logs" @@ -25,4 +27,4 @@ kafka_offsets_topic_segmant_bytes: 104857600 kafka_num_io_threads: 8 kafka_heap: 1G -repository_infrastructure: "{{ apache_mirror }}/kafka/{{ kafka_version }}" \ No newline at end of file +repository_infrastructure: "{{ apache_mirror }}/kafka/{{ kafka_version }}" diff --git a/roles/kafka/tasks/main.yml b/roles/kafka/tasks/main.yml index f1fbf56..36dc2a5 100644 --- a/roles/kafka/tasks/main.yml +++ b/roles/kafka/tasks/main.yml @@ -1,5 +1,18 @@ --- +- name: Configuring group + group: + name: "{{ kafka_group }}" + tags: kafka + +- name: create kafka user and group + user: + name={{ kafka_user }} + group={{ kafka_group }} + shell=/bin/bash + createhome=yes + tags: kafka + - name: create kafka directories file: path={{ item }} @@ -8,6 +21,7 @@ with_items: - "{{ kafka_install_dir }}" - "{{ kafka_data_log_dir }}" + - "{{ kafka_log_dir }}" tags: kafka - name: check for existing install @@ -88,4 +102,17 @@ line: "export KAFKA={{ kafka_install_dir }}/default" tags: env +- name: set owner and group of kafka directory and files + file: + path={{ item }} + state=directory + recurse=yes + owner={{ kafka_user }} + group={{ kafka_group }} + with_items: + - "{{ kafka_install_dir }}" + - "{{ kafka_data_dir }}" + - "{{ kafka_log_dir }}" + tags: kafka + # - cron: name="clear old kafka app logs" job="find /opt/kafka/default/logs -mtime +7 -exec rm {} \; > /dev/null" minute="0" diff --git a/roles/kafka/templates/kafka-supervisord.conf.j2 b/roles/kafka/templates/kafka-supervisord.conf.j2 index 4097b8c..43f9f5e 100644 --- a/roles/kafka/templates/kafka-supervisord.conf.j2 +++ b/roles/kafka/templates/kafka-supervisord.conf.j2 @@ -4,4 +4,5 @@ environment= KAFKA_HEAP_OPTS="-Xmx{{ kafka_heap }} -Xms{{ kafka_heap }}" autostart=true autorestart=true startsecs=5 +user={{ kafka_user }} stopsignal=KILL From 3414e9a0e040b54c125cd2e42029704847e86b25 Mon Sep 17 00:00:00 2001 From: David Cavitt Date: Fri, 15 Jul 2016 14:25:03 -0400 Subject: [PATCH 4/5] fixed missing kafka_user and kafka_group variables in kafka role --- roles/kafka/defaults/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/kafka/defaults/main.yml b/roles/kafka/defaults/main.yml index 0f6d0ca..f455b74 100644 --- a/roles/kafka/defaults/main.yml +++ b/roles/kafka/defaults/main.yml @@ -14,6 +14,8 @@ kafka_data_dir: kafka_data_log_dir: - "/opt/kafka/topic-logs" +kafka_user: kafka +kafka_group: kafka kafka_port: 9092 kafka_message_max: 10000000 kafka_replica_fetch_max_bytes: 15000000 From f8f42657a290479d60d2b1d5e014fd978c6cc207 Mon Sep 17 00:00:00 2001 From: Joseph McCullough Date: Tue, 26 Jul 2016 13:26:32 -0500 Subject: [PATCH 5/5] Change Elasticsearch DD tags to key-value pairs --- group_vars/cdr-elasticsearch-nodes | 2 +- group_vars/dig-elasticsearch-nodes | 2 +- group_vars/ist-elasticsearch-nodes | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/group_vars/cdr-elasticsearch-nodes b/group_vars/cdr-elasticsearch-nodes index b0fd46c..522cf12 100644 --- a/group_vars/cdr-elasticsearch-nodes +++ b/group_vars/cdr-elasticsearch-nodes @@ -12,4 +12,4 @@ datadog_monitors: is_custom_check: false es_instances: - url: http://localhost:9200 - tag: cdr-es-cluster + tag: "es-cluster:cdr" diff --git a/group_vars/dig-elasticsearch-nodes b/group_vars/dig-elasticsearch-nodes index a2138c0..ddeeee5 100644 --- a/group_vars/dig-elasticsearch-nodes +++ b/group_vars/dig-elasticsearch-nodes @@ -14,5 +14,5 @@ datadog_monitors: is_custom_check: false es_instances: - url: http://localhost:9200 - tag: dig-es-cluster + tag: "es-cluster:dig" diff --git a/group_vars/ist-elasticsearch-nodes b/group_vars/ist-elasticsearch-nodes index a52e97e..ddc4bc9 100644 --- a/group_vars/ist-elasticsearch-nodes +++ b/group_vars/ist-elasticsearch-nodes @@ -12,5 +12,5 @@ datadog_monitors: is_custom_check: false es_instances: - url: http://localhost:9200 - tag: ist-es-cluster + tag: "es-cluster:ist"