[8.1](backport #30068) ZooKeeper module: Adapt to ZooKeeper 3.6+ mntr response fields' changes.

CHANGELOG.next.asciidoc

@@ -56,6 +56,7 @@ https:/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Metricbeat*
 
+- Fixed missing ZooKeeper metrics due compatibility issues with versions >= 3.6.0 {pull}30068[30068]
 
 *Packetbeat*
 

metricbeat/docs/fields.asciidoc

@@ -63770,7 +63770,7 @@ type: long
 *`zookeeper.mntr.followers`*::
 +
 --
-Number of followers seen by the current host.
+Number of followers seen by the current host (Up to ZooKeeper 3.5.9).
 
 
 type: long
@@ -63915,6 +63915,16 @@ type: long
 Number of znodes reported by the local ZooKeeper process.
 
 
+type: long
+
+--
+
+*`zookeeper.mntr.learners`*::
++
+--
+Number of learners (either followers or observers) seen by the current host (From ZooKeeper 3.6.0)
+
+
 type: long
 
 --

metricbeat/docs/modules/zookeeper.asciidoc

@@ -17,9 +17,11 @@ metricsets are `mntr` and `server`.
 [float]
 === Compatibility
 
-The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version
+The ZooKeeper metricsets were tested with ZooKeeper 3.4.8, 3.6.0 and 3.7.0. They are expected to work with all versions
 >= 3.4.0. Versions prior to 3.4 do not support the `mntr` command.
 
+Note that from ZooKeeper 3.6.0, `mntr` command must be explicitly enabled at ZooKeeper side using the `4lw.commands.whitelist` configuration parameter.
+
 [float]
 === Dashboard
 

metricbeat/module/zookeeper/_meta/Dockerfile

@@ -1,3 +1,3 @@
 ARG ZOOKEEPER_VERSION
-FROM jplock/zookeeper:${ZOOKEEPER_VERSION}
+FROM zookeeper:${ZOOKEEPER_VERSION}
 HEALTHCHECK --interval=1s --retries=90 CMD nc -z localhost 2181

metricbeat/module/zookeeper/_meta/docs.asciidoc

@@ -8,9 +8,11 @@ metricsets are `mntr` and `server`.
 [float]
 === Compatibility
 
-The ZooKeeper metricsets were tested with ZooKeeper 3.4.8 and are expected to work with all version
+The ZooKeeper metricsets were tested with ZooKeeper 3.4.8, 3.6.0 and 3.7.0. They are expected to work with all versions
 >= 3.4.0. Versions prior to 3.4 do not support the `mntr` command.
 
+Note that from ZooKeeper 3.6.0, `mntr` command must be explicitly enabled at ZooKeeper side using the `4lw.commands.whitelist` configuration parameter.
+
 [float]
 === Dashboard
 

metricbeat/module/zookeeper/_meta/supported-versions.yml

@@ -0,0 +1,4 @@
+variants:
+ - ZOOKEEPER_VERSION: 3.5.5
+ - ZOOKEEPER_VERSION: 3.6.3
+ - ZOOKEEPER_VERSION: 3.7.0

metricbeat/module/zookeeper/docker-compose.yml

@@ -9,3 +9,5 @@ services:
  ZOOKEEPER_VERSION: ${ZOOKEEPER_VERSION:-3.5.5}
  ports:
  - 2181
+ environment:
+ - ZOO_4LW_COMMANDS_WHITELIST=stat, ruok, conf, isro, cons, mntr

metricbeat/module/zookeeper/mntr/_meta/fields.yml

@@ -24,7 +24,7 @@
  - name: followers
  type: long
  description: >
- Number of followers seen by the current host.
+ Number of followers seen by the current host (Up to ZooKeeper 3.5.9).
  - name: max_file_descriptor_count
  type: long
  description: >
@@ -82,3 +82,7 @@
  type: long
  description: >
  Number of znodes reported by the local ZooKeeper process.
+ - name: learners
+ type: long
+ description: >
+ Number of learners (either followers or observers) seen by the current host (From ZooKeeper 3.6.0)

metricbeat/module/zookeeper/mntr/data.go

@@ -36,9 +36,9 @@ var (
  schema = s.Schema{
  "version": c.Str("zk_version"),
  "latency": s.Object{
- "avg": c.Int("zk_avg_latency"),
- "min": c.Int("zk_min_latency"),
- "max": c.Int("zk_max_latency"),
+ "avg": c.Float("zk_avg_latency"),
+ "min": c.Float("zk_min_latency"),
+ "max": c.Float("zk_max_latency"),
  },
  "packets": s.Object{
  "received": c.Int("zk_packets_received"),
@@ -53,7 +53,8 @@ var (
  "approximate_data_size": c.Int("zk_approximate_data_size"),
  }
  schemaLeader = s.Schema{
- "followers": c.Int("zk_followers"),
+ "learners": c.Int("zk_learners", s.Optional),
+ "followers": c.Int("zk_followers", s.Optional), // Not present anymore in ZooKeeper >= 3.6 mntr responses
  "synced_followers": c.Int("zk_synced_followers"),
  "pending_syncs": c.Int("zk_pending_syncs"),
  }
@@ -85,9 +86,22 @@ func eventMapping(serverId string, response io.Reader, r mb.ReporterV2, logger *
  delete(event, "version")
  }
 
+ _, hasFollowers := fullEvent["zk_followers"]
+ _, hasLearners := fullEvent["zk_learners"]
+
  // only exposed by the Leader
- if _, ok := fullEvent["zk_followers"]; ok {
+ if hasLearners || hasFollowers {
  schemaLeader.ApplyTo(event, fullEvent)
+
+ // If ZK < 3.6, keep a migration to recent versions view of the "followers" field
+ if followers, ok := event["followers"]; ok {
+ event.Put("learners", followers)
+ }
+
+ // If ZK >= 3.6, keep a legacy view of the "learners" field
+ if learners, ok := event["learners"]; ok {
+ event.Put("followers", learners)
+ }
  }
 
  // only available on Unix platforms

metricbeat/module/zookeeper/mntr/mntr_integration_test.go

@@ -46,8 +46,8 @@ func TestFetch(t *testing.T) {
  e, _ := events[0].BeatEvent("zookeeper", "mntr").Fields.GetValue("zookeeper.mntr")
  event := e.(common.MapStr)
  // Check values
- avgLatency := event["latency"].(common.MapStr)["avg"].(int64)
- maxLatency := event["latency"].(common.MapStr)["max"].(int64)
+ avgLatency := event["latency"].(common.MapStr)["avg"].(float64)
+ maxLatency := event["latency"].(common.MapStr)["max"].(float64)
  numAliveConnections := event["num_alive_connections"].(int64)
 
  assert.True(t, avgLatency >= 0)

metricbeat/module/zookeeper/mntr/mntr_test.go

@@ -0,0 +1,111 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mntr
+
+import (
+ "bytes"
+ _ "embed"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+
+ "github.com/elastic/beats/v7/libbeat/common"
+ "github.com/elastic/beats/v7/libbeat/logp"
+ mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+)
+
+func assertExpectations(t *testing.T, expectations common.MapStr, report common.MapStr, message ...string) {
+ for key, expectation := range expectations {
+ assert.Contains(t, report, key, message)
+ switch expectation := expectation.(type) {
+ case common.MapStr:
+ nestedReport, _ := report.GetValue(key)
+ assert.IsType(t, nestedReport, report, message)
+ assertExpectations(t, expectation, nestedReport.(common.MapStr), message...)
+ default:
+ reportValue, _ := report.GetValue(key)
+ assert.Equal(t, expectation, reportValue, message)
+ }
+ }
+}
+
+//go:embed testdata/mntr.35.leader.txt
+var mntrTestInputZooKeeper35 string
+
+//go:embed testdata/mntr.37.leader.txt
+var mntrTestInputZooKeeper37 string
+
+func TestEventMapping(t *testing.T) {
+
+ type TestCase struct {
+ Version string
+ MntrSample string
+ ExpectedValues common.MapStr
+ }
+
+ mntrSamples := []TestCase{
+ {
+ "3.5.3",
+ mntrTestInputZooKeeper35,
+ common.MapStr{
+ "learners": int64(1),
+ "followers": int64(1),
+ "latency": common.MapStr{
+ "max": float64(29),
+ "avg": float64(0),
+ "min": float64(0),
+ },
+ },
+ },
+ {
+ "3.7.0",
+ mntrTestInputZooKeeper37,
+ common.MapStr{
+ "learners": int64(1),
+ "followers": int64(1),
+ "latency": common.MapStr{
+ "max": float64(8),
+ "avg": float64(0.5714),
+ "min": float64(0),
+ },
+ },
+ },
+ }
+
+ logger := logp.NewLogger("mntr_test")
+
+ for i, sample := range mntrSamples {
+ t.Run(sample.Version, func(t *testing.T) {
+
+ reporter := &mbtest.CapturingReporterV2{}
+
+ eventMapping(fmt.Sprint(i), bytes.NewReader([]byte(sample.MntrSample)), reporter, logger)
+
+ assert.Empty(t, reporter.GetErrors())
+
+ events := reporter.GetEvents()
+ assert.Len(t, events, 1)
+
+ event := events[len(events)-1]
+
+ assertExpectations(t, sample.ExpectedValues, event.MetricSetFields)
+ })
+ }
+
+}

metricbeat/module/zookeeper/mntr/testdata/mntr.35.leader.txt

@@ -0,0 +1,18 @@
+zk_version 3.5.3-beta-8ce24f9e675cbefffb8f21a47e06b42864475a60, built on 04/03/2017 16:19 GMT
+zk_avg_latency 0
+zk_max_latency 29
+zk_min_latency 0
+zk_packets_received 18835
+zk_packets_sent 19012
+zk_num_alive_connections 16
+zk_outstanding_requests 0
+zk_server_state leader
+zk_znode_count 489
+zk_watch_count 418
+zk_ephemerals_count 76
+zk_approximate_data_size 706622
+zk_open_file_descriptor_count 76
+zk_max_file_descriptor_count 1048576
+zk_followers 1
+zk_synced_followers 1
+zk_pending_syncs 0