Merge branch 'master' into PULSE-2498

istresearch · Jul 26, 2016 · d3abaf8 · d3abaf8
2 parents 609dc67 + b4d246a
commit d3abaf8
Show file tree

Hide file tree

Showing 8 changed files with 35 additions and 16 deletions.
diff --git a/group_vars/cdr-elasticsearch-nodes b/group_vars/cdr-elasticsearch-nodes
@@ -12,4 +12,4 @@ datadog_monitors:
     is_custom_check: false
     es_instances:
         - url: http://localhost:9200
-          tag: cdr-es-cluster
+          tag: "es-cluster:cdr"
diff --git a/group_vars/dig-elasticsearch-nodes b/group_vars/dig-elasticsearch-nodes
@@ -14,5 +14,5 @@ datadog_monitors:
     is_custom_check: false
     es_instances:
         - url: http://localhost:9200
-          tag: dig-es-cluster
+          tag: "es-cluster:dig"
 
diff --git a/group_vars/ist-elasticsearch-nodes b/group_vars/ist-elasticsearch-nodes
@@ -12,5 +12,5 @@ datadog_monitors:
     is_custom_check: false
     es_instances:
         - url: http://localhost:9200
-          tag: ist-es-cluster
+          tag: "es-cluster:ist"
 
diff --git a/group_vars/kafka-nodes b/group_vars/kafka-nodes
@@ -12,9 +12,17 @@ kafka_data_log_dir:
   - '/data/5/kafka/topic-logs'
   - '/data/6/kafka/topic-logs'
 
+kafka_data_dir:
+  - '/data/1/kafka'
+  - '/data/2/kafka'
+  - '/data/3/kafka'
+  - '/data/4/kafka'
+  - '/data/5/kafka'
+  - '/data/6/kafka'
+
 # According to Apache Docs, num io threads should be at least equal number of disks.
 kafka_num_io_threads: 6
 
 kafka_heap: 8G
 
-kafka_log_retention_hours: 24
+kafka_log_retention_hours: 24
diff --git a/host_vars/vagrant-ist-01/vars b/host_vars/vagrant-ist-01/vars
@@ -38,6 +38,10 @@ pulse_url_host: "{{ inventory_hostname }}"
 pulse_listen_port: 443
 
 kafka_heap: 1G
+kafka_data_log_dir:
+  - '/data/kafka/topic-logs'
+kafka_data_dir:
+  - '/data/kafka'
 
 sc_crawling_interface: eth0
 

diff --git a/roles/hadoop/tasks/main.yml b/roles/hadoop/tasks/main.yml
@@ -18,7 +18,7 @@
   file:
     path: "{{ item }}"
     state: directory
-    mode: 0754
+    mode: 0755
     owner: "{{ hdfs_user }}"
     group: "{{ hdfs_group }}"
   with_items:

diff --git a/roles/kafka/defaults/main.yml b/roles/kafka/defaults/main.yml
@@ -9,6 +9,8 @@ kafka_log_dir: /var/log/kafka
 kafka_log_rolling_retention_count: 10
 kafka_log_rolling_max_size: 25MB
 
+kafka_data_dir:
+  - "/opt/kafka"
 kafka_data_log_dir:
   - "/opt/kafka/topic-logs"
 

diff --git a/roles/kafka/tasks/main.yml b/roles/kafka/tasks/main.yml
@@ -1,15 +1,5 @@
 ---
 
-- name: create kafka directories
-  file:
-    path={{ item }}
-    state=directory
-    mode=0744
-  with_items:
-    - "{{ kafka_install_dir }}"
-    - "{{ kafka_data_log_dir }}"
-  tags: kafka
-
 - name: Configuring group
   group:
     name: "{{ kafka_group }}"
@@ -23,6 +13,17 @@
     createhome=yes
   tags: kafka
 
+- name: create kafka directories
+  file:
+    path={{ item }}
+    state=directory
+    mode=0744
+  with_items:
+    - "{{ kafka_install_dir }}"
+    - "{{ kafka_data_log_dir }}"
+    - "{{ kafka_log_dir }}"
+  tags: kafka
+
 - name: check for existing install
   stat: path={{ kafka_install_dir }}/kafka_{{ kafka_scala_version }}-{{ kafka_version }}
   register: kafka
@@ -103,11 +104,15 @@
 
 - name: set owner and group of kafka directory and files
   file:
-    path={{ kafka_install_dir }}
+    path={{ item }}
     state=directory
     recurse=yes
     owner={{ kafka_user }}
     group={{ kafka_group }}
+  with_items:
+    - "{{ kafka_install_dir }}"
+    - "{{ kafka_data_dir }}"
+    - "{{ kafka_log_dir }}"
   tags: kafka
 
 # - cron: name="clear old kafka app logs" job="find /opt/kafka/default/logs -mtime +7 -exec rm {} \; > /dev/null" minute="0"