Lower number of shards in the default configurations (elastic#5096)

This closes elastic#5095, by adding template settings in the default configuration files for each beat. (cherry picked from commit 307d5d6)
tsg · Sep 7, 2017 · 11a2e98 · 11a2e98
1 parent f18f97c
commit 11a2e98
Show file tree

Hide file tree

Showing 15 changed files with 88 additions and 3 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -60,17 +60,29 @@ https://github.com/elastic/beats/compare/v6.0.0-beta2...master[Check the HEAD di
 
 *Auditbeat*
 
+- Changed the number of shards in the default configuration to 3. {issue}5095[5095]
+
 *Filebeat*
 
+- Add PostgreSQL module with slowlog support. {pull}4763[4763]
+- Add Kafka log module. {pull}4885[4885]
+- Add support for `/var/log/containers/` log path in `add_kubernetes_metadata` processor. {pull}4981[4981]
+- Remove error log from runnerfactory as error is returned by API. {pull}5085[5085]
+- Changed the number of shards in the default configuration to 3. {issue}5095[5095]
+
 *Heartbeat*
 
+- Changed the number of shards in the default configuration to 1. {issue}5095[5095]
+
 *Metricbeat*
 
 *Packetbeat*
 
+- Changed the number of shards in the default configuration to 3. {issue}5095[5095]
+
 *Winlogbeat*
 
-*Auditbeat*
+- Changed the number of shards in the default configuration to 3. {issue}5095[5095]
 
 ==== Deprecated
 

diff --git a/auditbeat/Makefile b/auditbeat/Makefile
@@ -50,8 +50,9 @@ fields: python-env
 # Collects all module configs
 .PHONY: configs
 configs: python-env
-	@cat ${ES_BEATS}/auditbeat/_meta/common.yml \
-	     <(go run scripts/generate_config.go -os linux -concat) > _meta/beat.yml
+	@cat ${ES_BEATS}/auditbeat/_meta/common.p1.yml \
+	     <(go run scripts/generate_config.go -os linux -concat) \
+		 ${ES_BEATS}/auditbeat/_meta/common.p2.yml > _meta/beat.yml
 	@cat ${ES_BEATS}/auditbeat/_meta/common.reference.yml \
 	     <(go run scripts/generate_config.go -os linux -ref -concat) > _meta/beat.reference.yml
 

diff --git a/auditbeat/_meta/common.yml → auditbeat/_meta/common.p1.yml b/auditbeat/_meta/common.yml → auditbeat/_meta/common.p1.yml
diff --git a/auditbeat/_meta/common.p2.yml b/auditbeat/_meta/common.p2.yml
@@ -0,0 +1,6 @@
+
+#==================== Elasticsearch template setting ==========================
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml
@@ -29,6 +29,12 @@ auditbeat.modules:
 
 
 
+#==================== Elasticsearch template setting ==========================
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group

diff --git a/filebeat/_meta/common.p2.yml b/filebeat/_meta/common.p2.yml
@@ -65,3 +65,10 @@ filebeat.config.modules:
 
   # Period on which files under path should be checked for changes
   #reload.period: 10s
+
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml
@@ -75,6 +75,13 @@ filebeat.config.modules:
   # Period on which files under path should be checked for changes
   #reload.period: 10s
 
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group

diff --git a/heartbeat/_meta/beat.yml b/heartbeat/_meta/beat.yml
@@ -21,3 +21,10 @@ heartbeat.monitors:
 
   # Total test connection and data exchange timeout
   #timeout: 16s
+
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
diff --git a/heartbeat/heartbeat.yml b/heartbeat/heartbeat.yml
@@ -22,6 +22,13 @@ heartbeat.monitors:
   # Total test connection and data exchange timeout
   #timeout: 16s
 
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group

diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml
@@ -1,5 +1,6 @@
 
 #==========================  Modules configuration ============================
+
 metricbeat.config.modules:
   # Glob pattern for configuration loading
   path: ${path.config}/modules.d/*.yml
@@ -11,6 +12,7 @@ metricbeat.config.modules:
   #reload.period: 10s
 
 #==================== Elasticsearch template setting ==========================
+
 setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression

diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml
@@ -8,6 +8,7 @@
 # https://www.elastic.co/guide/en/beats/metricbeat/index.html
 
 #==========================  Modules configuration ============================
+
 metricbeat.config.modules:
   # Glob pattern for configuration loading
   path: ${path.config}/modules.d/*.yml
@@ -19,6 +20,7 @@ metricbeat.config.modules:
   #reload.period: 10s
 
 #==================== Elasticsearch template setting ==========================
+
 setup.template.settings:
   index.number_of_shards: 1
   index.codec: best_compression

diff --git a/packetbeat/_meta/beat.yml b/packetbeat/_meta/beat.yml
@@ -92,3 +92,10 @@ packetbeat.protocols:
   # Configure the ports where to listen for NFS traffic. You can disable
   # the NFS protocol by commenting out the list of ports.
   ports: [2049]
+
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml
@@ -93,6 +93,13 @@ packetbeat.protocols:
   # the NFS protocol by commenting out the list of ports.
   ports: [2049]
 
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group

diff --git a/winlogbeat/_meta/beat.yml b/winlogbeat/_meta/beat.yml
@@ -22,3 +22,10 @@ winlogbeat.event_logs:
     ignore_older: 72h
   - name: Security
   - name: System
+
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml
@@ -23,6 +23,13 @@ winlogbeat.event_logs:
   - name: Security
   - name: System
 
+#==================== Elasticsearch template setting ==========================
+
+setup.template.settings:
+  index.number_of_shards: 3
+  #index.codec: best_compression
+  #_source.enabled: false
+
 #================================ General =====================================
 
 # The name of the shipper that publishes the network data. It can be used to group