Backport of #15492 to 7.x: Add new page statistics to system/memory m…

…etricset (#15527) * [Metricbeat] Add new page statistics to system/memory metricset (#15492) * add new page statistics to system/memory * update field descriptions * make update * update data.json * update system tests * update changelog * fix conflicts * try to fix system python tests (cherry picked from commit 0fd2250) * update fields * fix readme
elastic · Jan 14, 2020 · 0f0a2d4 · 0f0a2d4
1 parent d1b3d3e
commit 0f0a2d4
Show file tree

Hide file tree

Showing 7 changed files with 194 additions and 18 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -145,6 +145,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Metricbeat*
 
+
+- Expand data for the `system/memory` metricset {pull}15492[15492]
 - Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342]
 - Add cost warnings for the azure module. {pull}15356[15356]
 - Release elb module as GA. {pull}15485[15485]

diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc
@@ -31684,6 +31684,89 @@ type: long
 The percentage of used swap memory.
 
 
+type: scaled_float
+
+format: percent
+
+--
+
+[float]
+=== page_stats
+
+memory page statistics
+
+
+*`system.memory.page_stats.pgscan_kswapd.pages`*::
++
+--
+pages scanned by kswapd
+
+type: long
+
+format: number
+
+--
+
+*`system.memory.page_stats.pgscan_direct.pages`*::
++
+--
+pages scanned directly
+
+type: long
+
+format: number
+
+--
+
+*`system.memory.page_stats.pgfree.pages`*::
++
+--
+pages freed by the system
+
+type: long
+
+format: number
+
+--
+
+*`system.memory.page_stats.pgsteal_kswapd.pages`*::
++
+--
+number of pages reclaimed by kswapd
+
+type: long
+
+format: number
+
+--
+
+*`system.memory.page_stats.pgsteal_direct.pages`*::
++
+--
+number of pages reclaimed directly
+
+type: long
+
+format: number
+
+--
+
+*`system.memory.page_stats.direct_efficiency.pct`*::
++
+--
+direct reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory.
+
+type: scaled_float
+
+format: percent
+
+--
+
+*`system.memory.page_stats.kswapd_efficiency.pct`*::
++
+--
+kswapd reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory.
+
 type: scaled_float
 
 format: percent

diff --git a/metricbeat/module/system/fields.go b/metricbeat/module/system/fields.go
diff --git a/metricbeat/module/system/memory/_meta/data.json b/metricbeat/module/system/memory/_meta/data.json
@@ -6,21 +6,22 @@
         "module": "system"
     },
     "metricset": {
-        "name": "memory"
+        "name": "memory",
+        "period": 10000
     },
     "service": {
         "type": "system"
     },
     "system": {
         "memory": {
             "actual": {
-                "free": 671084544,
+                "free": 14247317504,
                 "used": {
-                    "bytes": 362037248,
-                    "pct": 0.3504
+                    "bytes": 1407057920,
+                    "pct": 0.0899
                 }
             },
-            "free": 340848640,
+            "free": 4859097088,
             "hugepages": {
                 "default_size": 2097152,
                 "free": 0,
@@ -38,28 +39,51 @@
                     "pct": 0
                 }
             },
+            "page_stats": {
+                "direct_efficiency": {
+                    "pct": 0.9976
+                },
+                "kswapd_efficiency": {
+                    "pct": 0.6213
+                },
+                "pgfree": {
+                    "pages": 4382105954
+                },
+                "pgscan_direct": {
+                    "pages": 485820
+                },
+                "pgscan_kswapd": {
+                    "pages": 77390925
+                },
+                "pgsteal_direct": {
+                    "pages": 484631
+                },
+                "pgsteal_kswapd": {
+                    "pages": 48081976
+                }
+            },
             "swap": {
-                "free": 0,
+                "free": 7846490112,
                 "in": {
-                    "pages": 0
+                    "pages": 1111
                 },
                 "out": {
-                    "pages": 0
+                    "pages": 20255
                 },
                 "readahead": {
-                    "cached": 0,
-                    "pages": 0
+                    "cached": 28,
+                    "pages": 65
                 },
-                "total": 0,
+                "total": 7897870336,
                 "used": {
-                    "bytes": 0,
-                    "pct": 0
+                    "bytes": 51380224,
+                    "pct": 0.0065
                 }
             },
-            "total": 1033121792,
+            "total": 15654375424,
             "used": {
-                "bytes": 692273152,
-                "pct": 0.6701
+                "bytes": 10795278336,
+                "pct": 0.6896
             }
         }
     }

diff --git a/metricbeat/module/system/memory/_meta/fields.yml b/metricbeat/module/system/memory/_meta/fields.yml
@@ -102,6 +102,39 @@
           description: >
             The percentage of used swap memory.
 
+    - name: page_stats
+      type: group
+      description: memory page statistics
+      fields:
+        - name: pgscan_kswapd.pages
+          type: long
+          format: number
+          description: pages scanned by kswapd
+        - name: pgscan_direct.pages
+          type: long
+          format: number
+          description: pages scanned directly
+        - name: pgfree.pages
+          type: long
+          format: number
+          description: pages freed by the system
+        - name: pgsteal_kswapd.pages
+          type: long
+          format: number
+          description: number of pages reclaimed by kswapd
+        - name: pgsteal_direct.pages
+          type: long
+          format: number
+          description: number of pages reclaimed directly
+        - name: direct_efficiency.pct
+          type: scaled_float
+          format: percent
+          description: direct reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory.
+        - name: kswapd_efficiency.pct
+          type: scaled_float
+          format: percent
+          description: kswapd reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory.
+
     - name: hugepages
       type: group
       prefix: "[float]"

diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go
@@ -103,7 +103,38 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 			"pages":  vmstat.SwapRa,
 			"cached": vmstat.SwapRaHit,
 		}
+		pageStats := common.MapStr{
+			"pgscan_kswapd": common.MapStr{
+				"pages": vmstat.PgscanKswapd,
+			},
+			"pgscan_direct": common.MapStr{
+				"pages": vmstat.PgscanDirect,
+			},
+			"pgfree": common.MapStr{
+				"pages": vmstat.Pgfree,
+			},
+			"pgsteal_kswapd": common.MapStr{
+				"pages": vmstat.PgstealKswapd,
+			},
+			"pgsteal_direct": common.MapStr{
+				"pages": vmstat.PgstealDirect,
+			},
+		}
+		// This is similar to the vmeff stat gathered by sar
+		// these ratios calculate thhe efficiency of page reclaim
+		if vmstat.PgscanDirect != 0 {
+			pageStats["direct_efficiency"] = common.MapStr{
+				"pct": common.Round(float64(vmstat.PgstealDirect)/float64(vmstat.PgscanDirect), common.DefaultDecimalPlacesCount),
+			}
+		}
+
+		if vmstat.PgscanKswapd != 0 {
+			pageStats["kswapd_efficiency"] = common.MapStr{
+				"pct": common.Round(float64(vmstat.PgstealKswapd)/float64(vmstat.PgscanKswapd), common.DefaultDecimalPlacesCount),
+			}
+		}
 
+		memory["page_stats"] = pageStats
 	}
 
 	memory["swap"] = swap

diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py
@@ -42,7 +42,7 @@
 SYSTEM_FSSTAT_FIELDS = ["count", "total_files", "total_size"]
 
 SYSTEM_MEMORY_FIELDS = ["swap", "actual.free", "free", "total", "used.bytes", "used.pct", "actual.used.bytes",
-                        "actual.used.pct", "hugepages"]
+                        "actual.used.pct", "hugepages", "page_stats"]
 
 SYSTEM_NETWORK_FIELDS = ["name", "out.bytes", "in.bytes", "out.packets",
                          "in.packets", "in.error", "out.error", "in.dropped", "out.dropped"]
@@ -294,6 +294,9 @@ def test_memory(self):
         if not re.match("(?i)linux", sys.platform) and not "hugepages" in memory:
             # Ensure presence of hugepages only in Linux
             memory["hugepages"] = None
+        if not re.match("(?i)linux", sys.platform) and not "page_stats" in memory:
+            # Ensure presence of page_stats only in Linux
+            memory["page_stats"] = None
         self.assertItemsEqual(self.de_dot(SYSTEM_MEMORY_FIELDS), memory.keys())
 
         # Check that percentages are calculated.