ZooKeeper module: Adapt to ZooKeeper 3.6+ mntr response fields' cha…

…nges. (#30068)

CHANGELOG.next.asciidoc

@@ -61,6 +61,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...main[Check the HEAD dif
 *Metricbeat*
 
 - Enhance metricbeat on openshift documentation {pull}30054[30054]
+- Fixed missing ZooKeeper metrics due compatibility issues with versions >= 3.6.0 {pull}30068[30068]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -63984,7 +63984,7 @@ type: long
 *`zookeeper.mntr.followers`*::
 +
 --
-Number of followers seen by the current host.
+Number of followers seen by the current host (Up to ZooKeeper 3.5.9).
 
 
 type: long
@@ -64129,6 +64129,16 @@ type: long
 Number of znodes reported by the local ZooKeeper process.
 
 
+type: long
+
+--
+
+*`zookeeper.mntr.learners`*::
++
+--
+Number of learners (either followers or observers) seen by the current host (From ZooKeeper 3.6.0)
+
+
 type: long
 
 --

metricbeat/docs/modules/zookeeper.asciidoc

@@ -17,9 +17,11 @@ metricsets are `mntr` and `server`.
 [float]
 === Compatibility
 
-The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version
+The ZooKeeper metricsets were tested with ZooKeeper 3.4.8, 3.6.0 and 3.7.0. They are expected to work with all versions
 >= 3.4.0. Versions prior to 3.4 do not support the `mntr` command.
 
+Note that from ZooKeeper 3.6.0, `mntr` command must be explicitly enabled at ZooKeeper side using the `4lw.commands.whitelist` configuration parameter.
+
 [float]
 === Dashboard
 

metricbeat/module/zookeeper/_meta/Dockerfile

@@ -1,3 +1,3 @@
 ARG ZOOKEEPER_VERSION
-FROM jplock/zookeeper:${ZOOKEEPER_VERSION}
+FROM zookeeper:${ZOOKEEPER_VERSION}
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 2181

metricbeat/module/zookeeper/_meta/docs.asciidoc

@@ -8,9 +8,11 @@ metricsets are `mntr` and `server`.
 [float]
 === Compatibility
 
-The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version
+The ZooKeeper metricsets were tested with ZooKeeper 3.4.8, 3.6.0 and 3.7.0. They are expected to work with all versions
 >= 3.4.0. Versions prior to 3.4 do not support the `mntr` command.
 
+Note that from ZooKeeper 3.6.0, `mntr` command must be explicitly enabled at ZooKeeper side using the `4lw.commands.whitelist` configuration parameter.
+
 [float]
 === Dashboard
 

metricbeat/module/zookeeper/_meta/supported-versions.yml

@@ -0,0 +1,4 @@
+variants:
+  - ZOOKEEPER_VERSION: 3.5.5
+  - ZOOKEEPER_VERSION: 3.6.3
+  - ZOOKEEPER_VERSION: 3.7.0

metricbeat/module/zookeeper/docker-compose.yml

@@ -9,3 +9,5 @@ services:
         ZOOKEEPER_VERSION: ${ZOOKEEPER_VERSION:-3.5.5}
     ports:
       - 2181
+    environment:
+      - ZOO_4LW_COMMANDS_WHITELIST=stat, ruok, conf, isro, cons, mntr

metricbeat/module/zookeeper/mntr/_meta/fields.yml

@@ -24,7 +24,7 @@
     - name: followers
       type: long
       description: >
-        Number of followers seen by the current host.
+        Number of followers seen by the current host (Up to ZooKeeper 3.5.9).
     - name: max_file_descriptor_count
       type: long
       description: >
@@ -82,3 +82,7 @@
       type: long
       description: >
         Number of znodes reported by the local ZooKeeper process.
+    - name: learners
+      type: long
+      description: >
+        Number of learners (either followers or observers) seen by the current host (From ZooKeeper 3.6.0)

metricbeat/module/zookeeper/mntr/data.go

@@ -36,9 +36,9 @@ var (
 	schema       = s.Schema{
 		"version": c.Str("zk_version"),
 		"latency": s.Object{
-			"avg": c.Int("zk_avg_latency"),
-			"min": c.Int("zk_min_latency"),
-			"max": c.Int("zk_max_latency"),
+			"avg": c.Float("zk_avg_latency"),
+			"min": c.Float("zk_min_latency"),
+			"max": c.Float("zk_max_latency"),
 		},
 		"packets": s.Object{
 			"received": c.Int("zk_packets_received"),
@@ -53,7 +53,8 @@ var (
 		"approximate_data_size": c.Int("zk_approximate_data_size"),
 	}
 	schemaLeader = s.Schema{
-		"followers":        c.Int("zk_followers"),
+		"learners":         c.Int("zk_learners", s.Optional),
+		"followers":        c.Int("zk_followers", s.Optional), // Not present anymore in ZooKeeper >= 3.6 mntr responses
 		"synced_followers": c.Int("zk_synced_followers"),
 		"pending_syncs":    c.Int("zk_pending_syncs"),
 	}
@@ -85,9 +86,22 @@ func eventMapping(serverId string, response io.Reader, r mb.ReporterV2, logger *
 		delete(event, "version")
 	}
 
+	_, hasFollowers := fullEvent["zk_followers"]
+	_, hasLearners := fullEvent["zk_learners"]
+
 	// only exposed by the Leader
-	if _, ok := fullEvent["zk_followers"]; ok {
+	if hasLearners || hasFollowers {
 		schemaLeader.ApplyTo(event, fullEvent)
+
+		// If ZK < 3.6, keep a migration to recent versions view of the "followers" field
+		if followers, ok := event["followers"]; ok {
+			event.Put("learners", followers)
+		}
+
+		// If ZK >= 3.6, keep a legacy view of the "learners" field
+		if learners, ok := event["learners"]; ok {
+			event.Put("followers", learners)
+		}
 	}
 
 	// only available on Unix platforms

metricbeat/module/zookeeper/mntr/mntr_integration_test.go

@@ -46,8 +46,8 @@ func TestFetch(t *testing.T) {
 	e, _ := events[0].BeatEvent("zookeeper", "mntr").Fields.GetValue("zookeeper.mntr")
 	event := e.(common.MapStr)
 	// Check values
-	avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
-	maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+	avgLatency := event["latency"].(common.MapStr)["avg"].(float64)
+	maxLatency := event["latency"].(common.MapStr)["max"].(float64)
 	numAliveConnections := event["num_alive_connections"].(int64)
 
 	assert.True(t, avgLatency >= 0)

metricbeat/module/zookeeper/mntr/mntr_test.go

@@ -0,0 +1,111 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mntr
+
+import (
+	"bytes"
+	_ "embed"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+)
+
+func assertExpectations(t *testing.T, expectations common.MapStr, report common.MapStr, message ...string) {
+	for key, expectation := range expectations {
+		assert.Contains(t, report, key, message)
+		switch expectation := expectation.(type) {
+		case common.MapStr:
+			nestedReport, _ := report.GetValue(key)
+			assert.IsType(t, nestedReport, report, message)
+			assertExpectations(t, expectation, nestedReport.(common.MapStr), message...)
+		default:
+			reportValue, _ := report.GetValue(key)
+			assert.Equal(t, expectation, reportValue, message)
+		}
+	}
+}
+
+//go:embed testdata/mntr.35.leader.txt
+var mntrTestInputZooKeeper35 string
+
+//go:embed testdata/mntr.37.leader.txt
+var mntrTestInputZooKeeper37 string
+
+func TestEventMapping(t *testing.T) {
+
+	type TestCase struct {
+		Version        string
+		MntrSample     string
+		ExpectedValues common.MapStr
+	}
+
+	mntrSamples := []TestCase{
+		{
+			"3.5.3",
+			mntrTestInputZooKeeper35,
+			common.MapStr{
+				"learners":  int64(1),
+				"followers": int64(1),
+				"latency": common.MapStr{
+					"max": float64(29),
+					"avg": float64(0),
+					"min": float64(0),
+				},
+			},
+		},
+		{
+			"3.7.0",
+			mntrTestInputZooKeeper37,
+			common.MapStr{
+				"learners":  int64(1),
+				"followers": int64(1),
+				"latency": common.MapStr{
+					"max": float64(8),
+					"avg": float64(0.5714),
+					"min": float64(0),
+				},
+			},
+		},
+	}
+
+	logger := logp.NewLogger("mntr_test")
+
+	for i, sample := range mntrSamples {
+		t.Run(sample.Version, func(t *testing.T) {
+
+			reporter := &mbtest.CapturingReporterV2{}
+
+			eventMapping(fmt.Sprint(i), bytes.NewReader([]byte(sample.MntrSample)), reporter, logger)
+
+			assert.Empty(t, reporter.GetErrors())
+
+			events := reporter.GetEvents()
+			assert.Len(t, events, 1)
+
+			event := events[len(events)-1]
+
+			assertExpectations(t, sample.ExpectedValues, event.MetricSetFields)
+		})
+	}
+
+}

metricbeat/module/zookeeper/mntr/testdata/mntr.35.leader.txt

@@ -0,0 +1,18 @@
+zk_version	3.5.3-beta-8ce24f9e675cbefffb8f21a47e06b42864475a60, built on 04/03/2017 16:19 GMT
+zk_avg_latency	0
+zk_max_latency	29
+zk_min_latency	0
+zk_packets_received	18835
+zk_packets_sent	19012
+zk_num_alive_connections	16
+zk_outstanding_requests	0
+zk_server_state	leader
+zk_znode_count	489
+zk_watch_count	418
+zk_ephemerals_count	76
+zk_approximate_data_size	706622
+zk_open_file_descriptor_count	76
+zk_max_file_descriptor_count	1048576
+zk_followers	1
+zk_synced_followers	1
+zk_pending_syncs	0