Add redis key metricset (elastic#9657)

Add redis key metricset to collect information about keys
like length, type or TTL.

CHANGELOG.asciidoc

@@ -231,6 +231,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha1...v7.0.0-alpha2[Check the
 - Support GET requests in Jolokia module. {issue}8566[8566] {pull}9226[9226]
 - Add freebsd support for the uptime metricset. {pull}9413[9413]
 - Add `host.os.name` field to add_host_metadata processor. {issue}8948[8948] {pull}9405[9405]
+- Add `key` metricset to the Redis module. {issue}9582[9582] {pull}9657[9657]
 - Add more TCP statuses to `socket_summary` metricset. {pull}9430[9430]
 - Remove experimental tag from ceph metricsets. {pull}9708[9708]
 

metricbeat/docs/fields.asciidoc

@@ -20340,6 +20340,53 @@ type: long
 Count of slow operations
 
 
+--
+
+[float]
+== key fields
+
+`key` contains information about keys.
+
+
+
+*`redis.key.name`*::
++
+--
+type: keyword
+
+Key name.
+
+
+--
+
+*`redis.key.type`*::
++
+--
+type: keyword
+
+Key type as shown by `TYPE` command.
+
+
+--
+
+*`redis.key.length`*::
++
+--
+type: long
+
+Length of the key (Number of elements for lists, length for strings, cardinality for sets).
+
+
+--
+
+*`redis.key.expire.ttl`*::
++
+--
+type: long
+
+Seconds to expire.
+
+
 --
 
 [float]

metricbeat/docs/modules/redis.asciidoc

@@ -81,9 +81,13 @@ The following metricsets are available:
 
 * <<metricbeat-metricset-redis-info,info>>
 
+* <<metricbeat-metricset-redis-key,key>>
+
 * <<metricbeat-metricset-redis-keyspace,keyspace>>
 
 include::redis/info.asciidoc[]
 
+include::redis/key.asciidoc[]
+
 include::redis/keyspace.asciidoc[]
 

metricbeat/docs/modules/redis/key.asciidoc

@@ -0,0 +1,21 @@
+////
+This file is generated! See scripts/docs_collector.py
+////
+
+[[metricbeat-metricset-redis-key]]
+=== Redis key metricset
+
+include::../../../module/redis/key/_meta/docs.asciidoc[]
+
+
+==== Fields
+
+For a description of each field in the metricset, see the
+<<exported-fields-redis,exported fields>> section.
+
+Here is an example document generated by this metricset:
+
+[source,json]
+----
+include::../../../module/redis/key/_meta/data.json[]
+----

metricbeat/docs/modules_list.asciidoc

@@ -119,7 +119,8 @@ This file is generated! See scripts/docs_collector.py
 |<<metricbeat-metricset-rabbitmq-node,node>> beta[]  
 |<<metricbeat-metricset-rabbitmq-queue,queue>> beta[]  
 |<<metricbeat-module-redis,Redis>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
-.2+| .2+|  |<<metricbeat-metricset-redis-info,info>>   
+.3+| .3+|  |<<metricbeat-metricset-redis-info,info>>   
+|<<metricbeat-metricset-redis-key,key>>   
 |<<metricbeat-metricset-redis-keyspace,keyspace>>   
 |<<metricbeat-module-system,System>>     |image:./images/icon-yes.png[Prebuilt dashboards are available]    |  
 .14+| .14+|  |<<metricbeat-metricset-system-core,core>>   

metricbeat/module/redis/key/_meta/data.json

@@ -0,0 +1,32 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "agent": {
+        "hostname": "host.example.com",
+        "name": "host.example.com"
+    },
+    "event": {
+        "dataset": "redis.key",
+        "duration": 115000,
+        "module": "redis"
+    },
+    "metricset": {
+        "name": "key"
+    },
+    "redis": {
+        "key": {
+            "expire": {
+                "ttl": 360
+            },
+            "length": 3,
+            "name": "foo",
+            "type": "string"
+        },
+        "keyspace": {
+            "id": "db0"
+        }
+    },
+    "service": {
+        "address": "192.168.32.2:6379",
+        "type": "redis"
+    }
+}

metricbeat/module/redis/key/_meta/docs.asciidoc

@@ -0,0 +1,25 @@
+The Redis `key` metricset collects information about Redis keys.
+
+For each key matching one of the configured patterns, an event is sent to
+Elasticsearch with information about this key, what includes the type, its
+length when available, and its ttl.
+
+Patterns are configured as a list containing these fields:
+* `pattern` (required): pattern for key names, as accepted by the Redis
+  `KEYS` or `SCAN` commands.
+* `limit` (optional): safeguard when using patterns with wildcards to avoid
+  collecting too many keys (Default: 0, no limit)
+* `keyspace` (optional): Identifier of the database to use to look for the keys
+  (Default: 0)
+
+For example the following configuration will collect information about all keys
+whose name starts with `pipeline-*`, with a limit of 20 keys.
+
+[source,yaml]
+------------------------------------------------------------------------------
+- module: redis
+  metricsets: ['key']
+  key.patterns:
+    - name: 'pipeline-*'
+      limit: 20
+------------------------------------------------------------------------------

metricbeat/module/redis/key/_meta/fields.yml

@@ -0,0 +1,25 @@
+- name: key
+  type: group
+  description: >
+    `key` contains information about keys.
+  release: ga
+  fields:
+    - name: name
+      type: keyword
+      description: >
+        Key name.
+
+    - name: type
+      type: keyword
+      description: >
+        Key type as shown by `TYPE` command.
+
+    - name: length
+      type: long
+      description: >
+        Length of the key (Number of elements for lists, length for strings, cardinality for sets).
+
+    - name: expire.ttl
+      type: long
+      description: >
+        Seconds to expire.

metricbeat/module/redis/key/data.go

@@ -0,0 +1,36 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package key
+
+import (
+	"fmt"
+
+	"github.com/elastic/beats/libbeat/common"
+	"github.com/elastic/beats/metricbeat/mb"
+)
+
+func eventMapping(r mb.ReporterV2, keyspace uint, info map[string]interface{}) {
+	r.Event(mb.Event{
+		MetricSetFields: info,
+		ModuleFields: common.MapStr{
+			"keyspace": common.MapStr{
+				"id": fmt.Sprintf("db%d", keyspace),
+			},
+		},
+	})
+}

metricbeat/module/redis/key/key.go

@@ -0,0 +1,115 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package key
+
+import (
+	"time"
+
+	"github.com/elastic/beats/libbeat/logp"
+	"github.com/elastic/beats/metricbeat/mb"
+	"github.com/elastic/beats/metricbeat/mb/parse"
+	"github.com/elastic/beats/metricbeat/module/redis"
+
+	rd "github.com/garyburd/redigo/redis"
+)
+
+var (
+	debugf = logp.MakeDebug("redis-key")
+)
+
+func init() {
+	mb.Registry.MustAddMetricSet("redis", "key", New,
+		mb.WithHostParser(parse.PassThruHostParser),
+	)
+}
+
+// MetricSet for fetching Redis server information and statistics.
+type MetricSet struct {
+	mb.BaseMetricSet
+	pool     *rd.Pool
+	patterns []KeyPattern
+}
+
+// KeyPattern contains the information required to query keys
+type KeyPattern struct {
+	Keyspace uint   `config:"keyspace"`
+	Pattern  string `config:"pattern" validate:"required"`
+	Limit    uint   `config:"limit"`
+}
+
+// New creates new instance of MetricSet
+func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
+	// Unpack additional configuration options.
+	config := struct {
+		IdleTimeout time.Duration `config:"idle_timeout"`
+		Network     string        `config:"network"`
+		MaxConn     int           `config:"maxconn" validate:"min=1"`
+		Password    string        `config:"password"`
+		Patterns    []KeyPattern  `config:"key.patterns" validate:"nonzero,required"`
+	}{
+		Network:  "tcp",
+		MaxConn:  10,
+		Password: "",
+	}
+	err := base.Module().UnpackConfig(&config)
+	if err != nil {
+		return nil, err
+	}
+
+	return &MetricSet{
+		BaseMetricSet: base,
+		pool: redis.CreatePool(base.Host(), config.Password, config.Network,
+			config.MaxConn, config.IdleTimeout, base.Module().Config().Timeout),
+		patterns: config.Patterns,
+	}, nil
+}
+
+// Fetch fetches information from Redis keys
+func (m *MetricSet) Fetch(r mb.ReporterV2) {
+	conn := m.pool.Get()
+	for _, p := range m.patterns {
+		if err := redis.Select(conn, p.Keyspace); err != nil {
+			logp.Err("Failed to select keyspace %d: %s", p.Keyspace, err)
+			continue
+		}
+
+		keys, err := redis.FetchKeys(conn, p.Pattern, p.Limit)
+		if err != nil {
+			logp.Err("Failed to fetch list of keys in keyspace %d with pattern '%s': %s", p.Keyspace, p.Pattern, err)
+			continue
+		}
+		if p.Limit > 0 && len(keys) > int(p.Limit) {
+			debugf("Collecting stats for %d keys, but there are more available for pattern '%s' in keyspace %d", p.Limit)
+			keys = keys[:p.Limit]
+		}
+
+		for _, key := range keys {
+			keyInfo, err := redis.FetchKeyInfo(conn, key)
+			if err != nil {
+				logp.Err("Failed to fetch key info for key %s in keyspace %d", key, p.Keyspace)
+				continue
+			}
+			eventMapping(r, p.Keyspace, keyInfo)
+		}
+	}
+}
+
+// Close connections
+func (m *MetricSet) Close() error {
+	return m.pool.Close()
+}