Change Metricbeat default config to reduce disk space (#4329)

This implements several changes from #4112: * reduce the frequency for the fsstats and filesystem * filter out some filesystems by mount point * set one shard by default * enable best_compression by default
elastic · May 29, 2017 · 2f8d7bc · 2f8d7bc
1 parent 6964a1a
commit 2f8d7bc
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 78 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -74,6 +74,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 - Add `kube-state-metrics` based metrics to `kubernetes` module {pull}4253[4253]
 - Add debug logging to Jolokia JMX metricset. {pull}4341[4341]
 - Add events metricset for kubernetes metricbeat module {pull}4315[4315]
+- Change Metricbeat default configuration file to be better optimized for most users. {pull}4329[4329]
 
 *Packetbeat*
 

diff --git a/metricbeat/Makefile b/metricbeat/Makefile
@@ -53,6 +53,7 @@ collect-docs: python-env
 configs: python-env
 	cat ${ES_BEATS}/metricbeat/_meta/common.yml > _meta/beat.yml
 	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} $(PWD) >> _meta/beat.yml
+	cat ${ES_BEATS}/metricbeat/_meta/setup.yml >> _meta/beat.yml
 	cat ${ES_BEATS}/metricbeat/_meta/common.full.yml > _meta/beat.full.yml
 	. ${PYTHON_ENV}/bin/activate; python ${ES_BEATS}/script/config_collector.py --beat ${BEAT_NAME} --full $(PWD) >> _meta/beat.full.yml
 

diff --git a/metricbeat/_meta/setup.yml b/metricbeat/_meta/setup.yml
@@ -0,0 +1,5 @@
+#==================== Elasticsearch template setting ==========================
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
diff --git a/metricbeat/docs/modules/system.asciidoc b/metricbeat/docs/modules/system.asciidoc
@@ -86,45 +86,30 @@ in <<configuration-metricbeat>>. Here is an example configuration:
 ----
 metricbeat.modules:
 - module: system
+  enabled: true
+  period: 10s
   metricsets:
-    # CPU stats
     - cpu
-
-    # System Load stats
     - load
-
-    # Per CPU core stats
+    - memory
     #- core
-
-    # IO stats
     #- diskio
-
-    # Per filesystem stats
-    - filesystem
-
-    # File system summary stats
-    - fsstat
-
-    # Memory stats
-    - memory
-
-    # Network stats
     - network
-
-    # Processes summary
     - process_summary
-
-    # Per process stats
     - process
-
-    # Sockets (linux only)
     #- socket
-  enabled: true
-  period: 10s
   processes: ['.*']
   process.include_top_n:
     by_cpu: 5      # include top 5 processes by CPU
     by_memory: 5   # include top 5 processes by memory
+- module: system
+  enabled: true
+  period: 1m
+  metricsets:
+    - filesystem
+    - fsstat
+  filters:
+    - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
 ----
 
 [float]

diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml
@@ -12,47 +12,37 @@ metricbeat.modules:
 
 #------------------------------- System Module -------------------------------
 - module: system
+  enabled: true
+  period: 10s
   metricsets:
-    # CPU stats
     - cpu
-
-    # System Load stats
     - load
-
-    # Per CPU core stats
+    - memory
     #- core
-
-    # IO stats
     #- diskio
-
-    # Per filesystem stats
-    - filesystem
-
-    # File system summary stats
-    - fsstat
-
-    # Memory stats
-    - memory
-
-    # Network stats
     - network
-
-    # Processes summary
     - process_summary
-
-    # Per process stats
     - process
-
-    # Sockets (linux only)
     #- socket
-  enabled: true
-  period: 10s
   processes: ['.*']
   process.include_top_n:
     by_cpu: 5      # include top 5 processes by CPU
     by_memory: 5   # include top 5 processes by memory
+- module: system
+  enabled: true
+  period: 1m
+  metricsets:
+    - filesystem
+    - fsstat
+  filters:
+    - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
 
 
+#==================== Elasticsearch template setting ==========================
+setup.template.settings:
+  index.number_of_shards: 1
+  index.codec: best_compression
+  #_source.enabled: false
 
 #================================ General =====================================
 

diff --git a/metricbeat/module/system/_meta/config.yml b/metricbeat/module/system/_meta/config.yml
@@ -1,40 +1,25 @@
 - module: system
+  enabled: true
+  period: 10s
   metricsets:
-    # CPU stats
     - cpu
-
-    # System Load stats
     - load
-
-    # Per CPU core stats
+    - memory
     #- core
-
-    # IO stats
     #- diskio
-
-    # Per filesystem stats
-    - filesystem
-
-    # File system summary stats
-    - fsstat
-
-    # Memory stats
-    - memory
-
-    # Network stats
     - network
-
-    # Processes summary
     - process_summary
-
-    # Per process stats
     - process
-
-    # Sockets (linux only)
     #- socket
-  enabled: true
-  period: 10s
   processes: ['.*']
   process.include_top_n:
     by_cpu: 5      # include top 5 processes by CPU
     by_memory: 5   # include top 5 processes by memory
+- module: system
+  enabled: true
+  period: 1m
+  metricsets:
+    - filesystem
+    - fsstat
+  filters:
+    - drop_event.when.regexp.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
diff --git a/setup.yml b/setup.yml
@@ -0,0 +1 @@
+#======================== Template options ===============================
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		#======================== Template options ===============================