Remove deprecated kafka fields

CHANGELOG.next.asciidoc

@@ -89,6 +89,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Removes old module name aliases (gsuite) and removing old cyberark module in favor of the new cyberarkpas{pull}27915[27915]
 - Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
 - All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762]
+- Remove deprecated fields in Kafka module. {pull}27938[27938]
 
 *Heartbeat*
 - Remove long deprecated `watch_poll` functionality. {pull}27166[27166]
@@ -115,6 +116,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509]
 - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804]
 - Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933]
+- Remove deprecated fields in Kafka module. {pull}27938[27938]
 
 *Packetbeat*
 

filebeat/docs/fields.asciidoc

@@ -85190,24 +85190,6 @@ Kafka log lines.
 
 
 
-*`kafka.log.level`*::
-+
---
-type: alias
-
-alias to: log.level
-
---
-
-*`kafka.log.message`*::
-+
---
-type: alias
-
-alias to: message
-
---
-
 *`kafka.log.component`*::
 +
 --

filebeat/module/kafka/log/_meta/fields.yml

@@ -3,15 +3,6 @@
   description: >
     Kafka log lines.
   fields:
-    - name: level
-      type: alias
-      path: log.level
-      migration: true
-    - name: message
-      type: alias
-      path: message
-      migration: true
-
     - name: component
       type: keyword
       description: >

metricbeat/docs/fields.asciidoc

@@ -38022,33 +38022,6 @@ consumergroup
 
 
 
-[float]
-=== broker
-
-Broker Consumer Group Information have been read from (Broker handling the consumer group).
-
-
-
-*`kafka.consumergroup.broker.id`*::
-+
---
-Broker id
-
-
-type: long
-
---
-
-*`kafka.consumergroup.broker.address`*::
-+
---
-Broker address
-
-
-type: keyword
-
---
-
 *`kafka.consumergroup.id`*::
 +
 --
@@ -38058,30 +38031,6 @@ type: keyword
 
 --
 
-*`kafka.consumergroup.topic`*::
-+
---
-
-deprecated:[6.5]
-
-Topic name
-
-type: keyword
-
---
-
-*`kafka.consumergroup.partition`*::
-+
---
-
-deprecated:[6.5]
-
-Partition ID
-
-type: long
-
---
-
 *`kafka.consumergroup.offset`*::
 +
 --
@@ -38194,19 +38143,6 @@ Partition data.
 
 
 
-*`kafka.partition.partition.id`*::
-+
---
-
-deprecated:[6.5]
-
-Partition id.
-
-
-type: long
-
---
-
 *`kafka.partition.partition.leader`*::
 +
 --
@@ -38257,58 +38193,6 @@ type: long
 
 --
 
-*`kafka.partition.topic.error.code`*::
-+
---
-
-deprecated:[6.5]
-
-topic error code.
-
-
-type: long
-
---
-
-*`kafka.partition.topic.name`*::
-+
---
-
-deprecated:[6.5]
-
-Topic name
-
-
-type: keyword
-
---
-
-*`kafka.partition.broker.id`*::
-+
---
-
-deprecated:[6.5]
-
-Broker id
-
-
-type: long
-
---
-
-*`kafka.partition.broker.address`*::
-+
---
-
-deprecated:[6.5]
-
-Broker address
-
-
-type: keyword
-
---
-
 [float]
 === producer
 

metricbeat/module/kafka/consumergroup/_meta/data.json

@@ -7,28 +7,22 @@
     },
     "kafka": {
         "broker": {
-            "address": "localhost:32768",
+            "address": "172.21.0.2:9092",
             "id": 0
         },
         "consumergroup": {
-            "broker": {
-                "address": "localhost:32768",
-                "id": 0
-            },
             "client": {
                 "host": "127.0.0.1",
                 "id": "consumer-1",
-                "member_id": "consumer-1-a12ac7d4-00aa-45a0-8b35-0a9c6e880bf4"
+                "member_id": "consumer-1-8653cb3a-afed-4b1b-87d0-2a208319b41e"
             },
-            "consumer_lag": 1059,
+            "consumer_lag": 77,
             "error": {
                 "code": 0
             },
-            "id": "console-consumer-50413",
+            "id": "console-consumer-40539",
             "meta": "",
-            "offset": -1,
-            "partition": 0,
-            "topic": "test"
+            "offset": -1
         },
         "partition": {
             "id": 0,
@@ -43,7 +37,7 @@
         "period": 10000
     },
     "service": {
-        "address": "localhost:32768",
+        "address": "172.21.0.2:9092",
         "type": "kafka"
     }
 }

metricbeat/module/kafka/consumergroup/_meta/fields.yml

@@ -4,37 +4,10 @@
     consumergroup
   release: ga
   fields:
-    - name: broker
-      type: group
-      deprecated: 6.5
-      description: >
-        Broker Consumer Group Information have been read from (Broker handling
-        the consumer group).
-      fields:
-        - name: id
-          type: long
-          description: >
-            Broker id
-
-        - name: address
-          type: keyword
-          description: >
-            Broker address
-
     - name: id
       type: keyword
       description: Consumer Group ID
 
-    - name: topic
-      type: keyword
-      deprecated: 6.5
-      description: Topic name
-
-    - name: partition
-      type: long
-      deprecated: 6.5
-      description: Partition ID
-
     - name: offset
       type: long
       description: consumer offset into partition being read

metricbeat/module/kafka/consumergroup/consumergroup.go

@@ -94,19 +94,21 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 		// Helpful IDs to avoid scripts on queries
 		partitionTopicID := fmt.Sprintf("%d-%s", event["partition"], event["topic"])
 
-		// TODO (deprecation): Remove fields from MetricSetFields moved to ModuleFields
-		event["broker"] = brokerInfo
-		r.Event(mb.Event{
-			ModuleFields: common.MapStr{
-				"broker": brokerInfo,
-				"topic": common.MapStr{
-					"name": event["topic"],
-				},
-				"partition": common.MapStr{
-					"id":       event["partition"],
-					"topic_id": partitionTopicID,
-				},
+		moduleFields := common.MapStr{
+			"broker": brokerInfo,
+			"topic": common.MapStr{
+				"name": event["topic"],
+			},
+			"partition": common.MapStr{
+				"id":       event["partition"],
+				"topic_id": partitionTopicID,
 			},
+		}
+		delete(event, "topic")
+		delete(event, "partition")
+
+		r.Event(mb.Event{
+			ModuleFields:    moduleFields,
 			MetricSetFields: event,
 		})
 	}

metricbeat/module/kafka/partition/_meta/data.json

@@ -1,45 +1,40 @@
 {
     "@timestamp": "2017-10-12T08:05:34.853Z",
-    "beat": {
-        "hostname": "host.example.com",
-        "name": "host.example.com"
+    "event": {
+        "dataset": "kafka.partition",
+        "duration": 115000,
+        "module": "kafka"
     },
     "kafka": {
         "broker": {
-            "address": "172.18.0.2:9092",
+            "address": "172.21.0.2:9092",
             "id": 0
         },
         "partition": {
-            "broker": {
-                "address": "172.18.0.2:9092",
-                "id": 0
-            },
             "id": 0,
             "offset": {
-                "newest": 0,
+                "newest": 1,
                 "oldest": 0
             },
             "partition": {
-                "id": 0,
                 "insync_replica": true,
                 "is_leader": true,
                 "leader": 0,
                 "replica": 0
             },
-            "topic": {
-                "name": "foo-1538389014-739473801"
-            },
-            "topic_broker_id": "0-foo-1538389014-739473801-0",
-            "topic_id": "0-foo-1538389014-739473801"
+            "topic_broker_id": "0-metricbeat-generate-data-0",
+            "topic_id": "0-metricbeat-generate-data"
         },
         "topic": {
-            "name": "foo-1538389014-739473801"
+            "name": "metricbeat-generate-data"
         }
     },
     "metricset": {
-        "host": "kafka:9092",
-        "module": "kafka",
         "name": "partition",
-        "rtt": 115
+        "period": 10000
+    },
+    "service": {
+        "address": "172.21.0.2:9092",
+        "type": "kafka"
     }
 }