ZooKeeper module: Adapt to ZooKeeper 3.6+ mntr response fields' changes.

CHANGELOG.next.asciidoc

@@ -111,6 +111,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 - Extract correct index property in kibana.stats metricset {pull}29622[29622]
 - Fixed bug with `elasticsearch/cluster_stats` metricset not recording license expiration date correctly. {pull}29711[29711]
 - Enhance metricbeat on openshift documentation {pull}30054[30054]
+- Fixed missing ZooKeeper metrics due compatibility issues with versions >= 3.6.0 {pull}30068[30068]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -63770,7 +63770,7 @@ type: long
 *`zookeeper.mntr.followers`*::
 +
 --
-Number of followers seen by the current host.
+Number of followers seen by the current host (Up to ZooKeeper 3.5.9).
 
 
 type: long
@@ -63915,6 +63915,16 @@ type: long
 Number of znodes reported by the local ZooKeeper process.
 
 
+type: long
+
+--
+
+*`zookeeper.mntr.learners`*::
++
+--
+Number of learners (either followers or observers) seen by the current host (From ZooKeeper 3.6.0)
+
+
 type: long
 
 --

metricbeat/docs/modules/zookeeper.asciidoc

@@ -17,9 +17,11 @@ metricsets are `mntr` and `server`.
 [float]
 === Compatibility
 
-The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version
+The ZooKeeper metricsets were tested with ZooKeeper 3.4.8, 3.6.0 and 3.7.0. They are expected to work with all versions
 >= 3.4.0. Versions prior to 3.4 do not support the `mntr` command.
 
+Note that from ZooKeeper 3.6.0, `mntr` command must be explicitly enabled at ZooKeeper side using the `4lw.commands.whitelist` configuration parameter.
+
 [float]
 === Dashboard
 

metricbeat/module/zookeeper/_meta/Dockerfile

@@ -1,3 +1,3 @@
 ARG ZOOKEEPER_VERSION
-FROM jplock/zookeeper:${ZOOKEEPER_VERSION}
+FROM zookeeper:${ZOOKEEPER_VERSION}
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 2181

metricbeat/module/zookeeper/_meta/docs.asciidoc

@@ -8,9 +8,11 @@ metricsets are `mntr` and `server`.
 [float]
 === Compatibility
 
-The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version
+The ZooKeeper metricsets were tested with ZooKeeper 3.4.8, 3.6.0 and 3.7.0. They are expected to work with all versions
 >= 3.4.0. Versions prior to 3.4 do not support the `mntr` command.
 
+Note that from ZooKeeper 3.6.0, `mntr` command must be explicitly enabled at ZooKeeper side using the `4lw.commands.whitelist` configuration parameter.
+
 [float]
 === Dashboard
 

metricbeat/module/zookeeper/_meta/supported-versions.yml

@@ -0,0 +1,4 @@
+variants:
+  - ZOOKEEPER_VERSION: 3.5.5
+  - ZOOKEEPER_VERSION: 3.6.3
+  - ZOOKEEPER_VERSION: 3.7.0

metricbeat/module/zookeeper/docker-compose.yml

@@ -9,3 +9,5 @@ services:
         ZOOKEEPER_VERSION: ${ZOOKEEPER_VERSION:-3.5.5}
     ports:
       - 2181
+    environment:
+      - ZOO_4LW_COMMANDS_WHITELIST=stat, ruok, conf, isro, cons, mntr

metricbeat/module/zookeeper/mntr/_meta/fields.yml

@@ -24,7 +24,7 @@
     - name: followers
       type: long
       description: >
-        Number of followers seen by the current host.
+        Number of followers seen by the current host (Up to ZooKeeper 3.5.9).
     - name: max_file_descriptor_count
       type: long
       description: >
@@ -82,3 +82,7 @@
       type: long
       description: >
         Number of znodes reported by the local ZooKeeper process.
+    - name: learners
+      type: long
+      description: >
+        Number of learners (either followers or observers) seen by the current host (From ZooKeeper 3.6.0)

metricbeat/module/zookeeper/mntr/data.go

@@ -36,9 +36,9 @@ var (
 	schema       = s.Schema{
 		"version": c.Str("zk_version"),
 		"latency": s.Object{
-			"avg": c.Int("zk_avg_latency"),
-			"min": c.Int("zk_min_latency"),
-			"max": c.Int("zk_max_latency"),
+			"avg": c.Float("zk_avg_latency"),
+			"min": c.Float("zk_min_latency"),
+			"max": c.Float("zk_max_latency"),
 		},
 		"packets": s.Object{
 			"received": c.Int("zk_packets_received"),
@@ -53,7 +53,8 @@ var (
 		"approximate_data_size": c.Int("zk_approximate_data_size"),
 	}
 	schemaLeader = s.Schema{
-		"followers":        c.Int("zk_followers"),
+		"learners":         c.Int("zk_learners", s.Optional),
+		"followers":        c.Int("zk_followers", s.Optional), // Not present anymore in ZooKeeper >= 3.6 mntr responses
 		"synced_followers": c.Int("zk_synced_followers"),
 		"pending_syncs":    c.Int("zk_pending_syncs"),
 	}
@@ -86,8 +87,16 @@ func eventMapping(serverId string, response io.Reader, r mb.ReporterV2, logger *
 	}
 
 	// only exposed by the Leader
-	if _, ok := fullEvent["zk_followers"]; ok {
+	if hasFollowers, hasLearners := fullEvent["zk_followers"], fullEvent["zk_learners"]; hasFollowers != nil || hasLearners != nil {
 		schemaLeader.ApplyTo(event, fullEvent)
+
+		if learners, ok := event["learners"]; ok {
+			// If ZK >= 3.6, keep a legacy view of the "learners" field
+			event.Put("followers", learners)
+		} else if followers, ok := event["followers"]; ok {
+			// If ZK < 3.6, keep a migration to recent versions view of the "followers" field
+			event.Put("learners", followers)
+		}
 	}
 
 	// only available on Unix platforms

metricbeat/module/zookeeper/mntr/mntr_integration_test.go

@@ -46,8 +46,8 @@ func TestFetch(t *testing.T) {
 	e, _ := events[0].BeatEvent("zookeeper", "mntr").Fields.GetValue("zookeeper.mntr")
 	event := e.(common.MapStr)
 	// Check values
-	avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
-	maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+	avgLatency := event["latency"].(common.MapStr)["avg"].(float64)
+	maxLatency := event["latency"].(common.MapStr)["max"].(float64)
 	numAliveConnections := event["num_alive_connections"].(int64)
 
 	assert.True(t, avgLatency >= 0)

metricbeat/module/zookeeper/test_zookeeper.py

@@ -14,6 +14,7 @@
                "approximate_data_size", "num_alive_connections"]
 
 
+@metricbeat.parameterized_with_supported_versions
 class ZooKeeperMntrTest(metricbeat.BaseTest):
 
     COMPOSE_SERVICES = ['zookeeper']